Merge pull request #229 from safing/feature/ready-api

Ready API

.github/label-actions.yml

@@ -0,0 +1,40 @@
+# Configuration for Label Actions - https://github.com/dessant/label-actions
+
+community support:
+  comment: |
+    Hey @{issue-author}, thank you for raising this issue with us.
+
+    After a first review we noticed that this does not seem to be a technical issue, but rather a configuration issue or general question about how Portmaster works.
+
+    Thus, we invite the community to help with configuration and/or answering this questions.
+
+    If you are in a hurry or haven't received an answer, a good place to ask is in [our Discord community](https://discord.gg/safing).
+
+    If your problem or question has been resolved or answered, please come back and give an update here for other users encountering the same and then close this issue.
+
+    If you are a paying subscriber and want this issue to be checked out by Safing, please send us a message [on Discord](https://discord.gg/safing) or [via Email](mailto:support@safing.io) with your username and the link to this issue, so we can prioritize accordingly.
+
+needs debug info:
+  comment: |
+    Hey @{issue-author}, thank you for raising this issue with us.
+
+    After a first review we noticed that we will require the Debug Info for further investigation. However, you haven't supplied any Debug Info in your report.
+
+    Please [collect Debug Info](https://wiki.safing.io/en/FAQ/DebugInfo) from Portmaster _while_ the reported issue is present.
+
+in/compatibility:
+  comment: |
+    Hey @{issue-author}, thank you for reporting on a compatibility.
+
+    We keep a list of compatible software and user provided guides for improving compatibility [in the wiki - please have a look there](https://wiki.safing.io/en/Portmaster/App/Compatibility).
+    If you can't find your software in the list, then a good starting point is our guide on [How do I make software compatible with Portmaster](https://wiki.safing.io/en/FAQ/MakeSoftwareCompatibleWithPortmaster).
+
+    If you have managed to establish compatibility with an application, please share your findings here. This will greatly help other users encountering the same issues.
+
+fixed:
+  comment: |
+    This issue has been fixed by the recently referenced commit or PR.
+
+    However, the fix is not released yet.
+    
+    It is expected to go into the [Beta Release Channel](https://wiki.safing.io/en/FAQ/SwitchReleaseChannel) for testing within the next two weeks and will be available for everyone within the next four weeks. While this is the typical timeline we work with, things are subject to change.

.github/stale.yml

@@ -1,74 +0,0 @@
-# Configuration for probot-stale - https://github.com/probot/stale
-
-# Limit to only `issues` or `pulls`
-only: issues
-
-# Number of days of inactivity before an Issue or Pull Request becomes stale
-daysUntilStale: 21
-
-# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
-# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
-daysUntilClose: 7
-
-# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
-onlyLabels: []
-
-# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
-exemptLabels:
-  - "priority support"
-  - faq
-  - dependencies
-  - pinned
-  - security
-
-# Set to true to ignore issues in a project (defaults to false)
-exemptProjects: false
-
-# Set to true to ignore issues in a milestone (defaults to false)
-exemptMilestones: false
-
-# Set to true to ignore issues with an assignee (defaults to false)
-exemptAssignees: false
-
-# Label to use when marking as stale
-staleLabel: inactive
-
-# Comment to post when marking as stale. Set to `false` to disable
-markComment: >
-  This issue has been automatically marked as inactive because it has not had
-  recent activity. It will be closed if no further activity occurs. Thank you
-  for your contributions.
-
-# Comment to post when removing the stale label.
-# unmarkComment: >
-#   Your comment here.
-
-# Comment to post when closing a stale Issue or Pull Request.
-closeComment: >
-  This issue has been automatically closed because it has not had recent activity. Thank you for your contributions.
-
-  If the issue has not been resolved, you can find more information or continue the conversation here:
-
-  - [Docs & FAQ](https://docs.safing.io/)
-  - [Wiki](https://wiki.safing.io/)
-  - [Get Help on Discord](https://discord.gg/safing)
-
-  Please keep in mind that the free version of Portmaster only has limited support. We can only give so much limited free support.
-  If you find our work brings value to you, please consider supporting it by purchasing Supporter or Unlimited Packages https://safing.io/pricing/.
-
-  If you already are a paying subscriber and want to claim priority support for this issue, please send us a message [on Discord](https://discord.gg/safing) or [via Email](mailto:support@safing.io) with your username and the link to this issue, so we can prioritize accordingly.
-
-# Limit the number of actions per hour, from 1-30. Default is 30
-# limitPerRun: 30
-
-# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
-# pulls:
-#   daysUntilStale: 30
-#   markComment: >
-#     This pull request has been automatically marked as stale because it has not had
-#     recent activity. It will be closed if no further activity occurs. Thank you
-#     for your contributions.
-
-# issues:
-#   exemptLabels:
-#     - confirmed

.github/workflows/codeql-analysis.yml

@@ -1,72 +0,0 @@
-# For most projects, this workflow file will not need changing; you simply need
-# to commit it to your repository.
-#
-# You may wish to alter this file to override the set of languages analyzed,
-# or to provide custom queries or build logic.
-#
-# ******** NOTE ********
-# We have attempted to detect the languages in your repository. Please check
-# the `language` matrix defined below to confirm you have the correct set of
-# supported CodeQL languages.
-#
-name: "CodeQL"
-
-on:
-  push:
-    branches: [ "develop", master ]
-  pull_request:
-    # The branches below must be a subset of the branches above
-    branches: [ "develop" ]
-  schedule:
-    - cron: '17 17 * * 1'
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: [ 'go' ]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
-        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v3
-
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
-      with:
-        languages: ${{ matrix.language }}
-        # If you wish to specify custom queries, you can do so here or in a config file.
-        # By default, queries listed here will override any specified in a config file.
-        # Prefix the list here with "+" to use these queries and those in the config file.
-        
-        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
-        # queries: security-extended,security-and-quality
-
-        
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
-
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
-    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
-
-    # - run: |
-    #   echo "Run, Build Application using script"
-    #   ./location_of_script_within_repo/buildscript.sh
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2

.github/workflows/go.yml

@@ -21,7 +21,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v4
       with:
-        go-version: '^1.19'
+        go-version: '^1.21'
 
     - name: Get dependencies
       run: go mod download
@@ -46,7 +46,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v4
       with:
-        go-version: '^1.19'
+        go-version: '^1.21'
 
     - name: Get dependencies
       run: go mod download

.github/workflows/issues-first-greet.yml

@@ -0,0 +1,26 @@
+# This workflow responds to first time posters with a greeting message.
+# Docs: https://github.com/actions/first-interaction
+name: Greet New Users
+
+# This workflow is triggered when a new issue is created.
+on:
+  issues:
+    types: opened
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  greet:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/first-interaction@v1
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          # Respond to first time issue raisers.
+          issue-message: |
+            Greetings and welcome to our community! As this is the first issue you opened here, we wanted to share some useful infos with you:
+            
+            - 🗣️ Our community on [Discord](https://discord.gg/safing) is super helpful and active. We also have an AI-enabled support bot that knows Portmaster well and can give you immediate help.
+            - 📖 The [Wiki](https://wiki.safing.io/) answers all common questions and has many important details. If you can't find an answer there, let us know, so we can add anything that's missing.

.github/workflows/issues-label-actions.yml

@@ -0,0 +1,22 @@
+# This workflow responds with a message when certain labels are added to an issue or PR.
+# Docs: https://github.com/dessant/label-actions
+name: Label Actions
+
+# This workflow is triggered when a label is added to an issue.
+on:
+  issues:
+    types: labeled
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  action:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: dessant/label-actions@v3
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          config-path: ".github/label-actions.yml"
+          process-only: "issues"

.github/workflows/issues-stale.yml

@@ -0,0 +1,42 @@
+# This workflow warns and then closes stale issues and PRs.
+# Docs: https://github.com/actions/stale
+name: Close Stale Issues 
+
+on:
+  schedule:
+    - cron: "17 5 * * 1-5" # run at 5:17 (UTC) on Monday to Friday
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  issues: write
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          # Increase max operations.
+          # When using GITHUB_TOKEN, the rate limit is 1,000 requests per hour per repository.
+          operations-per-run: 500
+          # Handle stale issues
+          stale-issue-label: 'stale'
+          # Exemptions
+          exempt-all-issue-assignees: true
+          exempt-issue-labels: 'support,dependencies,pinned,security'
+          # Mark as stale
+          days-before-issue-stale: 63 # 2 months / 9 weeks
+          stale-issue-message: |
+            This issue has been automatically marked as inactive because it has not had activity in the past two months.
+            
+            If no further activity occurs, this issue will be automatically closed in one week in order to increase our focus on active topics.
+          # Close
+          days-before-issue-close: 7 # 1 week
+          close-issue-message: |
+            This issue has been automatically closed because it has not had recent activity. Thank you for your contributions.
+
+            If the issue has not been resolved, you can [find more information in our Wiki](https://wiki.safing.io/) or [continue the conversation on our Discord](https://discord.gg/safing).
+          # TODO: Handle stale PRs
+          days-before-pr-stale: 36500 # 100 years - effectively disabled.

api/authentication.go

@@ -151,7 +151,7 @@ func authenticateRequest(w http.ResponseWriter, r *http.Request, targetHandler h
 	switch requiredPermission { //nolint:exhaustive
 	case NotFound:
 		// Not found.
-		tracer.Trace("api: authenticated handler reported: not found")
+		tracer.Debug("api: no API endpoint registered for this path")
 		http.Error(w, "Not found.", http.StatusNotFound)
 		return nil
 	case NotSupported:

api/endpoints.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -15,6 +14,7 @@ import (
 	"github.com/gorilla/mux"
 
 	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/formats/dsd"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
 )
@@ -23,6 +23,13 @@ import (
 // Path and at least one permission are required.
 // As is exactly one function.
 type Endpoint struct { //nolint:maligned
+	// Name is the human reabable name of the endpoint.
+	Name string
+	// Description is the human readable description and documentation of the endpoint.
+	Description string
+	// Parameters is the parameter documentation.
+	Parameters []Parameter `json:",omitempty"`
+
 	// Path describes the URL path of the endpoint.
 	Path string
 
@@ -74,12 +81,6 @@ type Endpoint struct { //nolint:maligned
 
 	// HandlerFunc is the raw http handler.
 	HandlerFunc http.HandlerFunc `json:"-"`
-
-	// Documentation Metadata.
-
-	Name        string
-	Description string
-	Parameters  []Parameter `json:",omitempty"`
 }
 
 // Parameter describes a parameterized variation of an endpoint.
@@ -380,7 +381,7 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	// Wait for the owning module to be ready.
 	if !moduleIsReady(e.BelongsTo) {
-		http.Error(w, "The API endpoint is not ready yet or the its module is not enabled. Please try again later.", http.StatusServiceUnavailable)
+		http.Error(w, "The API endpoint is not ready yet or the its module is not enabled. Reload (F5) to try again.", http.StatusServiceUnavailable)
 		return
 	}
 
@@ -435,6 +436,9 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Add response headers to request struct so that the endpoint can work with them.
+	apiRequest.ResponseHeader = w.Header()
+
 	// Execute action function and get response data
 	var responseData []byte
 	var err error
@@ -457,7 +461,11 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		var v interface{}
 		v, err = e.StructFunc(apiRequest)
 		if err == nil && v != nil {
-			responseData, err = json.Marshal(v)
+			var mimeType string
+			responseData, mimeType, _, err = dsd.MimeDump(v, r.Header.Get("Accept"))
+			if err == nil {
+				w.Header().Set("Content-Type", mimeType)
+			}
 		}
 
 	case e.RecordFunc != nil:
@@ -478,7 +486,6 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 
 	// Check for handler error.
 	if err != nil {
-		// if statusProvider, ok := err.(HTTPStatusProvider); ok {
 		var statusProvider HTTPStatusProvider
 		if errors.As(err, &statusProvider) {
 			http.Error(w, err.Error(), statusProvider.HTTPStatus())
@@ -494,8 +501,12 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Set content type if not yet set.
+	if w.Header().Get("Content-Type") == "" {
+		w.Header().Set("Content-Type", e.MimeType+"; charset=utf-8")
+	}
+
 	// Write response.
-	w.Header().Set("Content-Type", e.MimeType+"; charset=utf-8")
 	w.Header().Set("Content-Length", strconv.Itoa(len(responseData)))
 	w.WriteHeader(http.StatusOK)
 	_, err = w.Write(responseData)

api/endpoints_debug.go

@@ -3,6 +3,7 @@ package api
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 	"os"
@@ -10,6 +11,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/safing/portbase/info"
+	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/utils/debug"
 )
 
@@ -24,6 +27,16 @@ func registerDebugEndpoints() error {
 		return err
 	}
 
+	if err := RegisterEndpoint(Endpoint{
+		Path:        "ready",
+		Read:        PermitAnyone,
+		ActionFunc:  ready,
+		Name:        "Ready",
+		Description: "Check if Portmaster has completed starting and is ready.",
+	}); err != nil {
+		return err
+	}
+
 	if err := RegisterEndpoint(Endpoint{
 		Path:        "debug/stack",
 		Read:        PermitAnyone,
@@ -46,6 +59,7 @@ func registerDebugEndpoints() error {
 
 	if err := RegisterEndpoint(Endpoint{
 		Path:     "debug/cpu",
+		MimeType: "application/octet-stream",
 		Read:     PermitAnyone,
 		DataFunc: handleCPUProfile,
 		Name:     "Get CPU Profile",
@@ -67,6 +81,7 @@ You can easily view this data in your browser with this command (with Go install
 
 	if err := RegisterEndpoint(Endpoint{
 		Path:     "debug/heap",
+		MimeType: "application/octet-stream",
 		Read:     PermitAnyone,
 		DataFunc: handleHeapProfile,
 		Name:     "Get Heap Profile",
@@ -81,6 +96,7 @@ You can easily view this data in your browser with this command (with Go install
 
 	if err := RegisterEndpoint(Endpoint{
 		Path:     "debug/allocs",
+		MimeType: "application/octet-stream",
 		Read:     PermitAnyone,
 		DataFunc: handleAllocsProfile,
 		Name:     "Get Allocs Profile",
@@ -114,9 +130,22 @@ You can easily view this data in your browser with this command (with Go install
 
 // ping responds with pong.
 func ping(ar *Request) (msg string, err error) {
+	// TODO: Remove upgrade to "ready" when all UI components have transitioned.
+	if modules.IsStarting() || modules.IsShuttingDown() {
+		return "", ErrorWithStatus(errors.New("portmaster is not ready, reload (F5) to try again"), http.StatusTooEarly)
+	}
+
 	return "Pong.", nil
 }
 
+// ready checks if Portmaster has completed starting.
+func ready(ar *Request) (msg string, err error) {
+	if modules.IsStarting() || modules.IsShuttingDown() {
+		return "", ErrorWithStatus(errors.New("portmaster is not ready, reload (F5) to try again"), http.StatusTooEarly)
+	}
+	return "Portmaster is ready.", nil
+}
+
 // getStack returns the current goroutine stack.
 func getStack(_ *Request) (data []byte, err error) {
 	buf := &bytes.Buffer{}
@@ -154,6 +183,12 @@ func handleCPUProfile(ar *Request) (data []byte, err error) {
 		duration = parsedDuration
 	}
 
+	// Indicate download and filename.
+	ar.ResponseHeader.Set(
+		"Content-Disposition",
+		fmt.Sprintf(`attachment; filename="portmaster-cpu-profile_v%s.pprof"`, info.Version()),
+	)
+
 	// Start CPU profiling.
 	buf := new(bytes.Buffer)
 	if err := pprof.StartCPUProfile(buf); err != nil {
@@ -175,6 +210,12 @@ func handleCPUProfile(ar *Request) (data []byte, err error) {
 
 // handleHeapProfile returns the Heap profile.
 func handleHeapProfile(ar *Request) (data []byte, err error) {
+	// Indicate download and filename.
+	ar.ResponseHeader.Set(
+		"Content-Disposition",
+		fmt.Sprintf(`attachment; filename="portmaster-memory-heap-profile_v%s.pprof"`, info.Version()),
+	)
+
 	buf := new(bytes.Buffer)
 	if err := pprof.Lookup("heap").WriteTo(buf, 0); err != nil {
 		return nil, fmt.Errorf("failed to write heap profile: %w", err)
@@ -184,6 +225,12 @@ func handleHeapProfile(ar *Request) (data []byte, err error) {
 
 // handleAllocsProfile returns the Allocs profile.
 func handleAllocsProfile(ar *Request) (data []byte, err error) {
+	// Indicate download and filename.
+	ar.ResponseHeader.Set(
+		"Content-Disposition",
+		fmt.Sprintf(`attachment; filename="portmaster-memory-allocs-profile_v%s.pprof"`, info.Version()),
+	)
+
 	buf := new(bytes.Buffer)
 	if err := pprof.Lookup("allocs").WriteTo(buf, 0); err != nil {
 		return nil, fmt.Errorf("failed to write allocs profile: %w", err)

api/request.go

@@ -26,6 +26,9 @@ type Request struct {
 	// AuthToken is the request-side authentication token assigned.
 	AuthToken *AuthToken
 
+	// ResponseHeader holds the response header.
+	ResponseHeader http.Header
+
 	// HandlerCache can be used by handlers to cache data between handlers within a request.
 	HandlerCache interface{}
 }

api/router.go

@@ -134,7 +134,7 @@ func (mh *mainHandler) handle(w http.ResponseWriter, r *http.Request) error {
 	}()
 
 	// Add security headers.
-	w.Header().Set("Referrer-Policy", "no-referrer")
+	w.Header().Set("Referrer-Policy", "same-origin")
 	w.Header().Set("X-Content-Type-Options", "nosniff")
 	w.Header().Set("X-Frame-Options", "deny")
 	w.Header().Set("X-XSS-Protection", "1; mode=block")
@@ -147,7 +147,7 @@ func (mh *mainHandler) handle(w http.ResponseWriter, r *http.Request) error {
 			"default-src 'self'; "+
 				"connect-src https://*.safing.io 'self'; "+
 				"style-src 'self' 'unsafe-inline'; "+
-				"img-src 'self' data:",
+				"img-src 'self' data: blob:",
 		)
 	}
 
@@ -235,6 +235,7 @@ func (mh *mainHandler) handle(w http.ResponseWriter, r *http.Request) error {
 		http.Error(lrw, "Method not allowed.", http.StatusMethodNotAllowed)
 		return nil
 	default:
+		tracer.Debug("api: no handler registered for this path")
 		http.Error(lrw, "Not found.", http.StatusNotFound)
 		return nil
 	}
@@ -270,7 +271,7 @@ func (mh *mainHandler) handle(w http.ResponseWriter, r *http.Request) error {
 	// Wait for the owning module to be ready.
 	if moduleHandler, ok := handler.(ModuleHandler); ok {
 		if !moduleIsReady(moduleHandler.BelongsTo()) {
-			http.Error(lrw, "The API endpoint is not ready yet. Please try again later.", http.StatusServiceUnavailable)
+			http.Error(lrw, "The API endpoint is not ready yet. Reload (F5) to try again.", http.StatusServiceUnavailable)
 			return nil
 		}
 	}

apprise/notify.go

@@ -0,0 +1,167 @@
+package apprise
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+
+	"github.com/safing/portbase/utils"
+)
+
+// Notifier sends messsages to an Apprise API.
+type Notifier struct {
+	// URL defines the Apprise API endpoint.
+	URL string
+
+	// DefaultType defines the default message type.
+	DefaultType MsgType
+
+	// DefaultTag defines the default message tag.
+	DefaultTag string
+
+	// DefaultFormat defines the default message format.
+	DefaultFormat MsgFormat
+
+	// AllowUntagged defines if untagged messages are allowed,
+	// which are sent to all configured apprise endpoints.
+	AllowUntagged bool
+
+	client     *http.Client
+	clientLock sync.Mutex
+}
+
+// Message represents the message to be sent to the Apprise API.
+type Message struct {
+	// Title is an optional title to go along with the body.
+	Title string `json:"title,omitempty"`
+
+	// Body is the main message content. This is the only required field.
+	Body string `json:"body"`
+
+	// Type defines the message type you want to send as.
+	// The valid options are info, success, warning, and failure.
+	// If no type is specified then info is the default value used.
+	Type MsgType `json:"type,omitempty"`
+
+	// Tag is used to notify only those tagged accordingly.
+	// Use a comma (,) to OR your tags and a space ( ) to AND them.
+	Tag string `json:"tag,omitempty"`
+
+	// Format optionally identifies the text format of the data you're feeding Apprise.
+	// The valid options are text, markdown, html.
+	// The default value if nothing is specified is text.
+	Format MsgFormat `json:"format,omitempty"`
+}
+
+// MsgType defines the message type.
+type MsgType string
+
+// Message Types.
+const (
+	TypeInfo    MsgType = "info"
+	TypeSuccess MsgType = "success"
+	TypeWarning MsgType = "warning"
+	TypeFailure MsgType = "failure"
+)
+
+// MsgFormat defines the message format.
+type MsgFormat string
+
+// Message Formats.
+const (
+	FormatText     MsgFormat = "text"
+	FormatMarkdown MsgFormat = "markdown"
+	FormatHTML     MsgFormat = "html"
+)
+
+type errorResponse struct {
+	Error string `json:"error"`
+}
+
+// Send sends a message to the Apprise API.
+func (n *Notifier) Send(ctx context.Context, m *Message) error {
+	// Check if the message has a body.
+	if m.Body == "" {
+		return errors.New("the message must have a body")
+	}
+
+	// Apply notifier defaults.
+	n.applyDefaults(m)
+
+	// Check if the message is tagged.
+	if m.Tag == "" && !n.AllowUntagged {
+		return errors.New("the message must have a tag")
+	}
+
+	// Marshal the message to JSON.
+	payload, err := json.Marshal(m)
+	if err != nil {
+		return fmt.Errorf("failed to marshal message: %w", err)
+	}
+
+	// Create request.
+	request, err := http.NewRequestWithContext(ctx, http.MethodPost, n.URL, bytes.NewReader(payload))
+	if err != nil {
+		return fmt.Errorf("failed to create request: %w", err)
+	}
+	request.Header.Set("Content-Type", "application/json")
+
+	// Send message to API.
+	resp, err := n.getClient().Do(request)
+	if err != nil {
+		return fmt.Errorf("failed to send message: %w", err)
+	}
+	defer resp.Body.Close() //nolint:errcheck,gosec
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusCreated, http.StatusNoContent, http.StatusAccepted:
+		return nil
+	default:
+		// Try to tease body contents.
+		if body, err := io.ReadAll(resp.Body); err == nil && len(body) > 0 {
+			// Try to parse json response.
+			errorResponse := &errorResponse{}
+			if err := json.Unmarshal(body, errorResponse); err == nil && errorResponse.Error != "" {
+				return fmt.Errorf("failed to send message: apprise returned %q with an error message: %s", resp.Status, errorResponse.Error)
+			}
+			return fmt.Errorf("failed to send message: %s (body teaser: %s)", resp.Status, utils.SafeFirst16Bytes(body))
+		}
+		return fmt.Errorf("failed to send message: %s", resp.Status)
+	}
+}
+
+func (n *Notifier) applyDefaults(m *Message) {
+	if m.Type == "" {
+		m.Type = n.DefaultType
+	}
+	if m.Tag == "" {
+		m.Tag = n.DefaultTag
+	}
+	if m.Format == "" {
+		m.Format = n.DefaultFormat
+	}
+}
+
+// SetClient sets a custom http client for accessing the Apprise API.
+func (n *Notifier) SetClient(client *http.Client) {
+	n.clientLock.Lock()
+	defer n.clientLock.Unlock()
+
+	n.client = client
+}
+
+func (n *Notifier) getClient() *http.Client {
+	n.clientLock.Lock()
+	defer n.clientLock.Unlock()
+
+	// Create client if needed.
+	if n.client == nil {
+		n.client = &http.Client{}
+	}
+
+	return n.client
+}

auto-comment.yml

@@ -1,23 +0,0 @@
-issueOpened: >
-  Thank you for reaching out.
-
-  In case you are raising an issue, you can find more information to try to it yourself here:
-
-  - [Wiki & FAQ](https://wiki.safing.io/)
-  - [GitHub Issues](https://github.com/issues?q=is%3Aopen+is%3Aissue+user%3Asafing+sort%3Aupdated-desc)
-  - [Ask on Discord](https://discord.gg/safing)
-
-  Additionally, there is a __ChatGPT-like support bot__ trained on our documentation, that you can [ask for help in this Discord channel](https://discord.com/channels/389815143711637517/1106170808704974878).
-
-  Please keep in mind that the free version of Portmaster only has community support and inactive issues are automatically closed after a while.
-  If you find our work brings value to you, please consider supporting it by purchasing Supporter or Unlimited Packages https://safing.io/pricing/.
-
-  If you are a customer, first of all: Thank You!
-  If you want to claim priority support for this issue, please send us a message [on Discord](https://discord.gg/safing) or [via Email](mailto:support@safing.io) with your username and the link to this issue, so we can prioritize accordingly.
-
-
-pullRequestOpened: >
-  Thank you for your pull request.
-
-  If you have not already, please read our [contribution guideline](https://wiki.safing.io/en/Contribute).
-  If this change is bigger and you have not discussed it with us, please head over to [Discord](https://discord.gg/safing) to discuss your idea.

config/basic_config.go

@@ -80,7 +80,7 @@ func registerBasicOptions() error {
 	// Register to hook to update the log level.
 	if err := module.RegisterEventHook(
 		"config",
-		configChangeEvent,
+		ChangeEvent,
 		"update log level",
 		setLogLevel,
 	); err != nil {

config/get_test.go

@@ -14,7 +14,7 @@ func parseAndReplaceConfig(jsonData string) error {
 		return err
 	}
 
-	validationErrors := replaceConfig(m)
+	validationErrors, _ := ReplaceConfig(m)
 	if len(validationErrors) > 0 {
 		return fmt.Errorf("%d errors, first: %w", len(validationErrors), validationErrors[0])
 	}
@@ -27,7 +27,7 @@ func parseAndReplaceDefaultConfig(jsonData string) error {
 		return err
 	}
 
-	validationErrors := replaceDefaultConfig(m)
+	validationErrors, _ := ReplaceDefaultConfig(m)
 	if len(validationErrors) > 0 {
 		return fmt.Errorf("%d errors, first: %w", len(validationErrors), validationErrors[0])
 	}

config/main.go

@@ -16,9 +16,8 @@ import (
 	"github.com/safing/portbase/utils/debug"
 )
 
-const (
-	configChangeEvent = "config change"
-)
+// ChangeEvent is the name of the config change event.
+const ChangeEvent = "config change"
 
 var (
 	module   *modules.Module
@@ -36,7 +35,7 @@ func SetDataRoot(root *utils.DirStructure) {
 
 func init() {
 	module = modules.Register("config", prep, start, nil, "database")
-	module.RegisterEvent(configChangeEvent, true)
+	module.RegisterEvent(ChangeEvent, true)
 
 	flag.BoolVar(&exportConfig, "export-config-options", false, "export configuration registry and exit")
 }
@@ -70,7 +69,7 @@ func start() error {
 
 	err = loadConfig(false)
 	if err != nil && !errors.Is(err, fs.ErrNotExist) {
-		return err
+		return fmt.Errorf("failed to load config file: %w", err)
 	}
 	return nil
 }

config/option.go

@@ -3,6 +3,7 @@ package config
 import (
 	"encoding/json"
 	"fmt"
+	"reflect"
 	"regexp"
 	"sync"
 
@@ -65,6 +66,9 @@ type PossibleValue struct {
 // Format: <vendor/package>:<scope>:<identifier> //.
 type Annotations map[string]interface{}
 
+// MigrationFunc is a function that migrates a config option value.
+type MigrationFunc func(option *Option, value any) any
+
 // Well known annotations defined by this package.
 const (
 	// DisplayHintAnnotation provides a hint for the user
@@ -108,10 +112,19 @@ const (
 	// requirement. The type of RequiresAnnotation is []ValueRequirement
 	// or ValueRequirement.
 	RequiresAnnotation = "safing/portbase:config:requires"
-	// RequiresFeaturePlan can be used to mark a setting as only available
+	// RequiresFeatureIDAnnotation can be used to mark a setting as only available
 	// when the user has a certain feature ID in the subscription plan.
 	// The type is []string or string.
-	RequiresFeatureID = "safing/portmaster:ui:config:requires-feature"
+	RequiresFeatureIDAnnotation = "safing/portmaster:ui:config:requires-feature"
+	// SettablePerAppAnnotation can be used to mark a setting as settable per-app and
+	// is a boolean.
+	SettablePerAppAnnotation = "safing/portmaster:settable-per-app"
+	// RequiresUIReloadAnnotation can be used to inform the UI that changing the value
+	// of the annotated setting requires a full reload of the user interface.
+	// The value of this annotation does not matter as the sole presence of
+	// the annotation key is enough. Though, users are advised to set the value
+	// of this annotation to true.
+	RequiresUIReloadAnnotation = "safing/portmaster:ui:requires-reload"
 )
 
 // QuickSettingsAction defines the action of a quick setting.
@@ -249,6 +262,9 @@ type Option struct {
 	// Annotations is considered mutable and setting/reading annotation keys
 	// must be performed while the option is locked.
 	Annotations Annotations
+	// Migrations holds migration functions that are given the raw option value
+	// before any validation is run. The returned value is then used.
+	Migrations []MigrationFunc `json:"-"`
 
 	activeValue         *valueCache // runtime value (loaded from config file or set by user)
 	activeDefaultValue  *valueCache // runtime default value (may be set internally)
@@ -301,6 +317,22 @@ func (option *Option) GetAnnotation(key string) (interface{}, bool) {
 	return val, ok
 }
 
+// AnnotationEquals returns whether the annotation of the given key matches the
+// given value.
+func (option *Option) AnnotationEquals(key string, value any) bool {
+	option.Lock()
+	defer option.Unlock()
+
+	if option.Annotations == nil {
+		return false
+	}
+	setValue, ok := option.Annotations[key]
+	if !ok {
+		return false
+	}
+	return reflect.DeepEqual(value, setValue)
+}
+
 // copyOrNil returns a copy of the option, or nil if copying failed.
 func (option *Option) copyOrNil() *Option {
 	copied, err := copystructure.Copy(option)
@@ -318,6 +350,30 @@ func (option *Option) IsSetByUser() bool {
 	return option.activeValue != nil
 }
 
+// UserValue returns the value set by the user or nil if the value has not
+// been changed from the default.
+func (option *Option) UserValue() any {
+	option.Lock()
+	defer option.Unlock()
+
+	if option.activeValue == nil {
+		return nil
+	}
+	return option.activeValue.getData(option)
+}
+
+// ValidateValue checks if the given value is valid for the option.
+func (option *Option) ValidateValue(value any) error {
+	option.Lock()
+	defer option.Unlock()
+
+	value = migrateValue(option, value)
+	if _, err := validateValue(option, value); err != nil {
+		return err
+	}
+	return nil
+}
+
 // Export expors an option to a Record.
 func (option *Option) Export() (record.Record, error) {
 	option.Lock()

config/persistence.go

@@ -45,7 +45,7 @@ func loadConfig(requireValidConfig bool) error {
 		return err
 	}
 
-	validationErrors := replaceConfig(newValues)
+	validationErrors, _ := ReplaceConfig(newValues)
 	if requireValidConfig && len(validationErrors) > 0 {
 		return fmt.Errorf("encountered %d validation errors during config loading", len(validationErrors))
 	}
@@ -58,10 +58,10 @@ func loadConfig(requireValidConfig bool) error {
 	return nil
 }
 
-// saveConfig saves the current configuration to file.
+// SaveConfig saves the current configuration to file.
 // It will acquire a read-lock on the global options registry
 // lock and must lock each option!
-func saveConfig() error {
+func SaveConfig() error {
 	optionsLock.RLock()
 	defer optionsLock.RUnlock()
 

config/perspective.go

@@ -35,6 +35,8 @@ optionsLoop:
 		if !ok {
 			continue
 		}
+		// migrate value
+		configValue = migrateValue(option, configValue)
 		// validate value
 		valueCache, err := validateValue(option, configValue)
 		if err != nil {

config/set.go

@@ -34,73 +34,118 @@ func signalChanges() {
 	validityFlag = abool.NewBool(true)
 	validityFlagLock.Unlock()
 
-	module.TriggerEvent(configChangeEvent, nil)
+	module.TriggerEvent(ChangeEvent, nil)
 }
 
-// replaceConfig sets the (prioritized) user defined config.
-func replaceConfig(newValues map[string]interface{}) []*ValidationError {
-	var validationErrors []*ValidationError
+// ValidateConfig validates the given configuration and returns all validation
+// errors as well as whether the given configuration contains unknown keys.
+func ValidateConfig(newValues map[string]interface{}) (validationErrors []*ValidationError, requiresRestart bool, containsUnknown bool) {
+	// RLock the options because we are not adding or removing
+	// options from the registration but rather only checking the
+	// options value which is guarded by the option's lock itself.
+	optionsLock.RLock()
+	defer optionsLock.RUnlock()
 
+	var checked int
+	for key, option := range options {
+		newValue, ok := newValues[key]
+		if ok {
+			checked++
+
+			func() {
+				option.Lock()
+				defer option.Unlock()
+
+				newValue = migrateValue(option, newValue)
+				_, err := validateValue(option, newValue)
+				if err != nil {
+					validationErrors = append(validationErrors, err)
+				}
+
+				if option.RequiresRestart {
+					requiresRestart = true
+				}
+			}()
+		}
+	}
+
+	return validationErrors, requiresRestart, checked < len(newValues)
+}
+
+// ReplaceConfig sets the (prioritized) user defined config.
+func ReplaceConfig(newValues map[string]interface{}) (validationErrors []*ValidationError, requiresRestart bool) {
 	// RLock the options because we are not adding or removing
 	// options from the registration but rather only update the
-	// options value which is guarded by the option's lock itself
+	// options value which is guarded by the option's lock itself.
 	optionsLock.RLock()
 	defer optionsLock.RUnlock()
 
 	for key, option := range options {
 		newValue, ok := newValues[key]
 
-		option.Lock()
-		option.activeValue = nil
+		func() {
+			option.Lock()
+			defer option.Unlock()
 
-		if ok {
-			valueCache, err := validateValue(option, newValue)
-			if err == nil {
-				option.activeValue = valueCache
-			} else {
-				validationErrors = append(validationErrors, err)
+			option.activeValue = nil
+			if ok {
+				newValue = migrateValue(option, newValue)
+				valueCache, err := validateValue(option, newValue)
+				if err == nil {
+					option.activeValue = valueCache
+				} else {
+					validationErrors = append(validationErrors, err)
+				}
 			}
-		}
+			handleOptionUpdate(option, true)
 
-		handleOptionUpdate(option, true)
-		option.Unlock()
+			if option.RequiresRestart {
+				requiresRestart = true
+			}
+		}()
 	}
 
 	signalChanges()
 
-	return validationErrors
+	return validationErrors, requiresRestart
 }
 
-// replaceDefaultConfig sets the (fallback) default config.
-func replaceDefaultConfig(newValues map[string]interface{}) []*ValidationError {
-	var validationErrors []*ValidationError
-
+// ReplaceDefaultConfig sets the (fallback) default config.
+func ReplaceDefaultConfig(newValues map[string]interface{}) (validationErrors []*ValidationError, requiresRestart bool) {
 	// RLock the options because we are not adding or removing
 	// options from the registration but rather only update the
-	// options value which is guarded by the option's lock itself
+	// options value which is guarded by the option's lock itself.
 	optionsLock.RLock()
 	defer optionsLock.RUnlock()
 
 	for key, option := range options {
 		newValue, ok := newValues[key]
 
-		option.Lock()
-		option.activeDefaultValue = nil
-		if ok {
-			valueCache, err := validateValue(option, newValue)
-			if err == nil {
-				option.activeDefaultValue = valueCache
-			} else {
-				validationErrors = append(validationErrors, err)
+		func() {
+			option.Lock()
+			defer option.Unlock()
+
+			option.activeDefaultValue = nil
+			if ok {
+				newValue = migrateValue(option, newValue)
+				valueCache, err := validateValue(option, newValue)
+				if err == nil {
+					option.activeDefaultValue = valueCache
+				} else {
+					validationErrors = append(validationErrors, err)
+				}
 			}
-		}
-		handleOptionUpdate(option, true)
-		option.Unlock()
+			handleOptionUpdate(option, true)
+
+			if option.RequiresRestart {
+				requiresRestart = true
+			}
+		}()
 	}
 
 	signalChanges()
 
-	return validationErrors
+	return validationErrors, requiresRestart
 }
 
 // SetConfigOption sets a single value in the (prioritized) user defined config.
@@ -118,6 +163,7 @@ func setConfigOption(key string, value any, push bool) (err error) {
 	if value == nil {
 		option.activeValue = nil
 	} else {
+		value = migrateValue(option, value)
 		valueCache, vErr := validateValue(option, value)
 		if vErr == nil {
 			option.activeValue = valueCache
@@ -141,7 +187,7 @@ func setConfigOption(key string, value any, push bool) (err error) {
 	// finalize change, activate triggers
 	signalChanges()
 
-	return saveConfig()
+	return SaveConfig()
 }
 
 // SetDefaultConfigOption sets a single value in the (fallback) default config.
@@ -159,6 +205,7 @@ func setDefaultConfigOption(key string, value interface{}, push bool) (err error
 	if value == nil {
 		option.activeDefaultValue = nil
 	} else {
+		value = migrateValue(option, value)
 		valueCache, vErr := validateValue(option, value)
 		if vErr == nil {
 			option.activeDefaultValue = valueCache

config/set_test.go

@@ -24,7 +24,7 @@ func TestLayersGetters(t *testing.T) { //nolint:paralleltest
 		t.Fatal(err)
 	}
 
-	validationErrors := replaceConfig(mapData)
+	validationErrors, _ := ReplaceConfig(mapData)
 	if len(validationErrors) > 0 {
 		t.Fatalf("%d errors, first: %s", len(validationErrors), validationErrors[0].Error())
 	}

config/validate.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"math"
 	"reflect"
+
+	"github.com/safing/portbase/log"
 )
 
 type valueCache struct {
@@ -64,6 +66,18 @@ func isAllowedPossibleValue(opt *Option, value interface{}) error {
 	return errors.New("value is not allowed")
 }
 
+// migrateValue runs all value migrations.
+func migrateValue(option *Option, value any) any {
+	for _, migration := range option.Migrations {
+		newValue := migration(option, value)
+		if newValue != value {
+			log.Debugf("config: migrated %s value from %v to %v", option.Key, value, newValue)
+		}
+		value = newValue
+	}
+	return value
+}
+
 // validateValue ensures that value matches the expected type of option.
 // It does not create a copy of the value!
 func validateValue(option *Option, value interface{}) (*valueCache, *ValidationError) { //nolint:gocyclo
@@ -76,8 +90,6 @@ func validateValue(option *Option, value interface{}) (*valueCache, *ValidationE
 		}
 	}
 
-	reflect.TypeOf(value).ConvertibleTo(reflect.TypeOf(""))
-
 	var validated *valueCache
 	switch v := value.(type) {
 	case string:

database/migration/migration.go

@@ -114,7 +114,7 @@ func (reg *Registry) Migrate(ctx context.Context) (err error) {
 		if err := m.MigrateFunc(migrationCtx, lastAppliedMigration, target, db); err != nil {
 			diag.Wrapped = err
 			diag.FailedMigration = m.Description
-			tracer.Infof("migration: applied migration for %s: %s - %s", reg.key, target.String(), m.Description)
+			tracer.Errorf("migration: migration for %s failed: %s - %s", reg.key, target.String(), m.Description)
 			tracer.Submit()
 			return diag
 		}

database/record/base.go

@@ -44,6 +44,13 @@ func (b *Base) SetKey(key string) {
 	}
 }
 
+// ResetKey resets the database name and key.
+// Use with caution!
+func (b *Base) ResetKey() {
+	b.dbName = ""
+	b.dbKey = ""
+}
+
 // Key returns the key of the database record.
 // As the key must be set before any usage and can only be set once, this
 // function may be used without locking the record.

formats/dsd/dsd.go

@@ -10,6 +10,7 @@ import (
 	"io"
 
 	"github.com/fxamacker/cbor/v2"
+	"github.com/ghodss/yaml"
 	"github.com/vmihailenco/msgpack/v5"
 
 	"github.com/safing/portbase/formats/varint"
@@ -41,6 +42,12 @@ func LoadAsFormat(data []byte, format uint8, t interface{}) (err error) {
 			return fmt.Errorf("dsd: failed to unpack json: %w, data: %s", err, utils.SafeFirst16Bytes(data))
 		}
 		return nil
+	case YAML:
+		err = yaml.Unmarshal(data, t)
+		if err != nil {
+			return fmt.Errorf("dsd: failed to unpack yaml: %w, data: %s", err, utils.SafeFirst16Bytes(data))
+		}
+		return nil
 	case CBOR:
 		err = cbor.Unmarshal(data, t)
 		if err != nil {
@@ -121,6 +128,11 @@ func dumpWithoutIdentifier(t interface{}, format uint8, indent string) ([]byte,
 		if err != nil {
 			return nil, err
 		}
+	case YAML:
+		data, err = yaml.Marshal(t)
+		if err != nil {
+			return nil, err
+		}
 	case CBOR:
 		data, err = cbor.Marshal(t)
 		if err != nil {

formats/dsd/format.go

@@ -19,6 +19,7 @@ const (
 	GenCode = 71 // G
 	JSON    = 74 // J
 	MsgPack = 77 // M
+	YAML    = 89 // Y
 
 	// Compression types.
 	GZIP = 90 // Z
@@ -48,6 +49,8 @@ func ValidateSerializationFormat(format uint8) (validatedFormat uint8, ok bool)
 		return format, true
 	case JSON:
 		return format, true
+	case YAML:
+		return format, true
 	case MsgPack:
 		return format, true
 	default:

formats/dsd/http.go

@@ -5,8 +5,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"mime"
 	"net/http"
+	"strings"
 )
 
 // HTTP Related Errors.
@@ -37,21 +37,8 @@ func loadFromHTTP(body io.Reader, mimeType string, t interface{}) (format uint8,
 		return 0, fmt.Errorf("dsd: failed to read http body: %w", err)
 	}
 
-	// Get mime type from header, then check, clean and verify it.
-	if mimeType == "" {
-		return 0, ErrMissingContentType
-	}
-	mimeType, _, err = mime.ParseMediaType(mimeType)
-	if err != nil {
-		return 0, fmt.Errorf("dsd: failed to parse content type: %w", err)
-	}
-	format, ok := MimeTypeToFormat[mimeType]
-	if !ok {
-		return 0, ErrIncompatibleFormat
-	}
-
-	// Parse data..
-	return format, LoadAsFormat(data, format, t)
+	// Load depending on mime type.
+	return MimeLoad(data, mimeType, t)
 }
 
 // RequestHTTPResponseFormat sets the Accept header to the given format.
@@ -61,11 +48,6 @@ func RequestHTTPResponseFormat(r *http.Request, format uint8) (mimeType string,
 	if !ok {
 		return "", ErrIncompatibleFormat
 	}
-	// Omit charset.
-	mimeType, _, err = mime.ParseMediaType(mimeType)
-	if err != nil {
-		return "", fmt.Errorf("dsd: failed to parse content type: %w", err)
-	}
 
 	// Request response format.
 	r.Header.Set("Accept", mimeType)
@@ -76,6 +58,7 @@ func RequestHTTPResponseFormat(r *http.Request, format uint8) (mimeType string,
 // DumpToHTTPRequest dumps the given data to the HTTP request using the given
 // format. It also sets the Accept header to the same format.
 func DumpToHTTPRequest(r *http.Request, t interface{}, format uint8) error {
+	// Get mime type and set request format.
 	mimeType, err := RequestHTTPResponseFormat(r, format)
 	if err != nil {
 		return err
@@ -87,7 +70,7 @@ func DumpToHTTPRequest(r *http.Request, t interface{}, format uint8) error {
 		return fmt.Errorf("dsd: failed to serialize: %w", err)
 	}
 
-	// Set body.
+	// Add data to request.
 	r.Header.Set("Content-Type", mimeType)
 	r.Body = io.NopCloser(bytes.NewReader(data))
 
@@ -97,16 +80,8 @@ func DumpToHTTPRequest(r *http.Request, t interface{}, format uint8) error {
 // DumpToHTTPResponse dumpts the given data to the HTTP response, using the
 // format defined in the request's Accept header.
 func DumpToHTTPResponse(w http.ResponseWriter, r *http.Request, t interface{}) error {
-	// Get format from Accept header.
-	// TODO: Improve parsing of Accept header.
-	mimeType := r.Header.Get("Accept")
-	format, ok := MimeTypeToFormat[mimeType]
-	if !ok {
-		return ErrIncompatibleFormat
-	}
-
-	// Serialize data.
-	data, err := dumpWithoutIdentifier(t, format, "")
+	// Serialize data based on accept header.
+	data, mimeType, _, err := MimeDump(t, r.Header.Get("Accept"))
 	if err != nil {
 		return fmt.Errorf("dsd: failed to serialize: %w", err)
 	}
@@ -120,16 +95,84 @@ func DumpToHTTPResponse(w http.ResponseWriter, r *http.Request, t interface{}) e
 	return nil
 }
 
+// MimeLoad loads the given data into the interface based on the given mime type accept header.
+func MimeLoad(data []byte, accept string, t interface{}) (format uint8, err error) {
+	// Find format.
+	format = FormatFromAccept(accept)
+	if format == 0 {
+		return 0, ErrIncompatibleFormat
+	}
+
+	// Load data.
+	err = LoadAsFormat(data, format, t)
+	return format, err
+}
+
+// MimeDump dumps the given interface based on the given mime type accept header.
+func MimeDump(t any, accept string) (data []byte, mimeType string, format uint8, err error) {
+	// Find format.
+	format = FormatFromAccept(accept)
+	if format == AUTO {
+		return nil, "", 0, ErrIncompatibleFormat
+	}
+
+	// Serialize and return.
+	data, err = dumpWithoutIdentifier(t, format, "")
+	return data, mimeType, format, err
+}
+
+// FormatFromAccept returns the format for the given accept definition.
+// The accept parameter matches the format of the HTTP Accept header.
+// Special cases, in this order:
+// - If accept is an empty string: returns default serialization format.
+// - If accept contains no supported format, but a wildcard: returns default serialization format.
+// - If accept contains no supported format, and no wildcard: returns AUTO format.
+func FormatFromAccept(accept string) (format uint8) {
+	if accept == "" {
+		return DefaultSerializationFormat
+	}
+
+	var foundWildcard bool
+	for _, mimeType := range strings.Split(accept, ",") {
+		// Clean mime type.
+		mimeType = strings.TrimSpace(mimeType)
+		mimeType, _, _ = strings.Cut(mimeType, ";")
+		if strings.Contains(mimeType, "/") {
+			_, mimeType, _ = strings.Cut(mimeType, "/")
+		}
+		mimeType = strings.ToLower(mimeType)
+
+		// Check if mime type is supported.
+		format, ok := MimeTypeToFormat[mimeType]
+		if ok {
+			return format
+		}
+
+		// Return default mime type as fallback if any mimetype is okay.
+		if mimeType == "*" {
+			foundWildcard = true
+		}
+	}
+
+	if foundWildcard {
+		return DefaultSerializationFormat
+	}
+	return AUTO
+}
+
 // Format and MimeType mappings.
 var (
 	FormatToMimeType = map[uint8]string{
-		JSON:    "application/json; charset=utf-8",
 		CBOR:    "application/cbor",
+		JSON:    "application/json",
 		MsgPack: "application/msgpack",
+		YAML:    "application/yaml",
 	}
 	MimeTypeToFormat = map[string]uint8{
-		"application/json":    JSON,
-		"application/cbor":    CBOR,
-		"application/msgpack": MsgPack,
+		"cbor":    CBOR,
+		"json":    JSON,
+		"msgpack": MsgPack,
+		"yaml":    YAML,
+		"yml":     YAML,
 	}
 )

formats/dsd/http_test.go

@@ -0,0 +1,45 @@
+package dsd
+
+import (
+	"mime"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMimeTypes(t *testing.T) {
+	t.Parallel()
+
+	// Test static maps.
+	for _, mimeType := range FormatToMimeType {
+		cleaned, _, err := mime.ParseMediaType(mimeType)
+		assert.NoError(t, err, "mime type must be parse-able")
+		assert.Equal(t, mimeType, cleaned, "mime type should be clean in map already")
+	}
+	for mimeType := range MimeTypeToFormat {
+		cleaned, _, err := mime.ParseMediaType(mimeType)
+		assert.NoError(t, err, "mime type must be parse-able")
+		assert.Equal(t, mimeType, cleaned, "mime type should be clean in map already")
+	}
+
+	// Test assumptions.
+	for accept, format := range map[string]uint8{
+		"application/json, image/webp":       JSON,
+		"image/webp, application/json":       JSON,
+		"application/json;q=0.9, image/webp": JSON,
+		"*":                                  DefaultSerializationFormat,
+		"*/*":                                DefaultSerializationFormat,
+		"text/yAMl":                          YAML,
+		" * , yaml ":                         YAML,
+		"yaml;charset ,*":                    YAML,
+		"xml,*":                              DefaultSerializationFormat,
+		"text/xml, text/other":               AUTO,
+		"text/*":                             DefaultSerializationFormat,
+		"yaml ;charset":                      AUTO, // Invalid mimetype format.
+		"":                                   DefaultSerializationFormat,
+		"x":                                  AUTO,
+	} {
+		derivedFormat := FormatFromAccept(accept)
+		assert.Equal(t, format, derivedFormat, "assumption for %q should hold", accept)
+	}
+}

go.mod

@@ -1,34 +1,37 @@
 module github.com/safing/portbase
 
-go 1.20
+go 1.21.1
+
+toolchain go1.21.2
 
 require (
-	github.com/VictoriaMetrics/metrics v1.24.0
+	github.com/VictoriaMetrics/metrics v1.29.0
 	github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6
 	github.com/armon/go-radix v1.0.0
 	github.com/bluele/gcache v0.0.2
 	github.com/davecgh/go-spew v1.1.1
 	github.com/dgraph-io/badger v1.6.2
-	github.com/fxamacker/cbor/v2 v2.4.0
+	github.com/fxamacker/cbor/v2 v2.5.0
+	github.com/ghodss/yaml v1.0.0
 	github.com/gofrs/uuid v4.4.0+incompatible
-	github.com/gorilla/mux v1.8.0
-	github.com/gorilla/websocket v1.5.0
+	github.com/gorilla/mux v1.8.1
+	github.com/gorilla/websocket v1.5.1
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/go-version v1.6.0
 	github.com/mitchellh/copystructure v1.2.0
-	github.com/safing/jess v0.3.1
-	github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626
+	github.com/safing/jess v0.3.3
+	github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec
 	github.com/seehuhn/fortuna v1.0.1
 	github.com/shirou/gopsutil v3.21.11+incompatible
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.4
 	github.com/tevino/abool v1.2.0
-	github.com/tidwall/gjson v1.15.0
+	github.com/tidwall/gjson v1.17.0
 	github.com/tidwall/sjson v1.2.5
-	github.com/vmihailenco/msgpack/v5 v5.3.5
-	go.etcd.io/bbolt v1.3.7
-	golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b
-	golang.org/x/sync v0.3.0
-	golang.org/x/sys v0.11.0
+	github.com/vmihailenco/msgpack/v5 v5.4.1
+	go.etcd.io/bbolt v1.3.8
+	golang.org/x/exp v0.0.0-20231219180239-dc181d75b848
+	golang.org/x/sync v0.5.0
+	golang.org/x/sys v0.15.0
 )
 
 require (
@@ -38,11 +41,12 @@ require (
 	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fxamacker/cbor v1.5.1 // indirect
-	github.com/go-ole/go-ole v1.2.6 // indirect
-	github.com/golang/glog v1.1.1 // indirect
+	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/golang/glog v1.2.0 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/btree v1.1.2 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.6 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/mr-tron/base58 v1.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
@@ -51,16 +55,19 @@ require (
 	github.com/seehuhn/sha256d v1.0.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
-	github.com/tklauser/go-sysconf v0.3.11 // indirect
-	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/tklauser/go-sysconf v0.3.13 // indirect
+	github.com/tklauser/numcpus v0.7.0 // indirect
 	github.com/valyala/fastrand v1.1.0 // indirect
 	github.com/valyala/histogram v1.2.0 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
-	golang.org/x/crypto v0.11.0 // indirect
-	golang.org/x/net v0.13.0 // indirect
-	golang.org/x/time v0.3.0 // indirect
-	google.golang.org/protobuf v1.31.0 // indirect
+	github.com/zeebo/blake3 v0.2.3 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gvisor.dev/gvisor v0.0.0-20231222013827-149350e5c428 // indirect
 )

go.sum

@@ -1,17 +1,13 @@
-github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
-github.com/VictoriaMetrics/metrics v1.22.2/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc=
-github.com/VictoriaMetrics/metrics v1.24.0 h1:ILavebReOjYctAGY5QU2F9X0MYvkcrG3aEn2RKa1Zkw=
-github.com/VictoriaMetrics/metrics v1.24.0/go.mod h1:eFT25kvsTidQFHb6U0oa0rTrDRdz4xTYjpL8+UPohys=
+github.com/VictoriaMetrics/metrics v1.29.0 h1:3qC+jcvymGJaQKt6wsXIlJieVFQwD/par9J1Bxul+Mc=
+github.com/VictoriaMetrics/metrics v1.29.0/go.mod h1:r7hveu6xMdUACXvB8TYdAj8WEsKzWB0EkpJN+RDtOf8=
 github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ=
 github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U=
 github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 h1:5L8Mj9Co9sJVgW3TpYk2gxGJnDjsYuboNTcRmbtGKGs=
 github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6/go.mod h1:3HgLJ9d18kXMLQlJvIY3+FszZYMxCz8WfE2MQ7hDY0w=
-github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -19,24 +15,18 @@ github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw=
 github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
-github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
-github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=
-github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
 github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
 github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
-github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
 github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
 github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
@@ -47,33 +37,31 @@ github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+m
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fxamacker/cbor v1.5.1 h1:XjQWBgdmQyqimslUh5r4tUGmoqzHmBFQOImkWGi2awg=
 github.com/fxamacker/cbor v1.5.1/go.mod h1:3aPGItF174ni7dDzd6JZ206H8cmr4GDNBGpPa971zsU=
-github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD88=
-github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
-github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
+github.com/fxamacker/cbor/v2 v2.5.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
-github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
-github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
+github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
 github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
 github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
-github.com/golang/glog v1.1.1 h1:jxpi2eWoU84wbX9iIEyAeeoac3FLuifZpY9tcNUD9kw=
-github.com/golang/glog v1.1.1/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
+github.com/golang/glog v1.2.0 h1:uCdmnmatrKCgMBlM4rMuJZWOkPDqdbZPnrMXDY4gI68=
+github.com/golang/glog v1.2.0/go.mod h1:6AhwSGph0fcJtXVM/PEHPqZlFeoLxhs7/t5UDAwmO+w=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
 github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
+github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
 github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -82,22 +70,16 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
+github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c=
+github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc=
+github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
-github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
-github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
-github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
-github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
 github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
 github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -113,14 +95,10 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/safing/jess v0.3.0/go.mod h1:JbYsPk5iJZx0OXDZeMcjS9qEdkGVUg+DCA8Fw2LdN9s=
-github.com/safing/jess v0.3.1 h1:cMZVhi2whW/YdD98MPLeLIWJndQ7o2QVt2HefQ/ByFA=
-github.com/safing/jess v0.3.1/go.mod h1:aj73Eot1zm2ETkJuw9hJlIO8bRom52uBbsCHemvlZmA=
-github.com/safing/portbase v0.15.2/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4fdlOMS91Y=
-github.com/safing/portbase v0.16.2/go.mod h1:mzNCWqPbO7vIYbbK5PElGbudwd2vx4YPNawymL8Aro8=
-github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626 h1:olc/REnUdpJN/Gmz8B030OxLpMYxyPDTrDILNEw0eKs=
-github.com/safing/portmaster-android/go v0.0.0-20230605085256-6abf4c495626/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE=
+github.com/safing/jess v0.3.3 h1:0U0bWdO0sFCgox+nMOqISFrnJpVmi+VFOW1xdX6q3qw=
+github.com/safing/jess v0.3.3/go.mod h1:t63qHB+4xd1HIv9MKN/qI2rc7ytvx7d6l4hbX7zxer0=
+github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec h1:oSJY1seobofPwpMoJRkCgXnTwfiQWNfGMCPDfqgAEfg=
+github.com/safing/portmaster-android/go v0.0.0-20230830120134-3226ceac3bec/go.mod h1:abwyAQrZGemWbSh/aCD9nnkp0SvFFf/mGWkAbOwPnFE=
 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/seehuhn/fortuna v1.0.1 h1:lu9+CHsmR0bZnx5Ay646XvCSRJ8PJTi5UYJwDBX68H0=
@@ -134,29 +112,19 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
-github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
-github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
-github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
-github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
 github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg=
 github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.15.0 h1:5n/pM+v3r5ujuNl4YLZLsQ+UE5jlkLVm7jMzT5Mpolw=
-github.com/tidwall/gjson v1.15.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
+github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM=
+github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
 github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
 github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
 github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
@@ -164,104 +132,70 @@ github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
 github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
 github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
-github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=
-github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk=
-github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
-github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
-github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8=
-github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ=
-github.com/tklauser/numcpus v0.5.0/go.mod h1:OGzpTxpcIMNGYQdit2BYL1pvk/dSOaJWjKoflh+RQjo=
-github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
-github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
-github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/tklauser/go-sysconf v0.3.13 h1:GBUpcahXSpR2xN01jhkNAbTLRk2Yzgggk8IM08lq3r4=
+github.com/tklauser/go-sysconf v0.3.13/go.mod h1:zwleP4Q4OehZHGn4CYZDipCgg9usW5IJePewFCGVEa0=
+github.com/tklauser/numcpus v0.7.0 h1:yjuerZP127QG9m5Zh/mSO4wqurYil27tHrqwRoRjpr4=
+github.com/tklauser/numcpus v0.7.0/go.mod h1:bb6dMVcj8A42tSE7i32fsIUCbQNllK5iDguyOZRUzAY=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8=
 github.com/valyala/fastrand v1.1.0/go.mod h1:HWqCzkrkg6QXT8V2EXWvXCoow7vLwOFN002oeRzjapQ=
 github.com/valyala/histogram v1.2.0 h1:wyYGAZZt3CpwUiIb9AU/Zbllg1llXyrtApRS815OLoQ=
 github.com/valyala/histogram v1.2.0/go.mod h1:Hb4kBwb4UxsaNbbbh+RRz8ZR6pdodR57tzWUS3BUzXY=
-github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
-github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
+github.com/vmihailenco/msgpack/v5 v5.4.1 h1:cQriyiUvjTwOHg8QZaPihLWeRAAVoCpE00IUPn0Bjt8=
+github.com/vmihailenco/msgpack/v5 v5.4.1/go.mod h1:GaZTsDaehaPpQVyxrf5mtQlH+pc21PIudVV/E3rRQok=
 github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
 github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
 github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM=
 github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
-github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
 github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-github.com/zalando/go-keyring v0.2.1/go.mod h1:g63M2PPn0w5vjmEbwAX3ib5I+41zdm4esSETOn9Y6Dw=
-go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
-go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ=
-go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.2.3 h1:TFoLXsjeXqRNFxSbk35Dk4YtszE/MQQGK10BH4ptoTg=
+github.com/zeebo/blake3 v0.2.3/go.mod h1:mjJjZpnsyIVtVgTOSpJ9vmRE4wgDeyt2HU3qXvvKCaQ=
+github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo=
+github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
+go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA=
+go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20221010152910-d6f0a8c073c2/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA=
-golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio=
-golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b h1:r+vk0EmXNmekl0S0BascoeeoHk/L7wmaW2QF90K+kYI=
-golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
-golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/exp v0.0.0-20231219180239-dc181d75b848 h1:+iq7lrkxmFNBM7xx+Rae2W6uyPfhPeDWD+n+JgppptE=
+golang.org/x/exp v0.0.0-20231219180239-dc181d75b848/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI=
+golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0=
+golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220927171203-f486391704dc/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
-golang.org/x/net v0.13.0 h1:Nvo8UFsZ8X3BhAC9699Z1j7XQ3rsZnUUm7jfBEk1ueY=
-golang.org/x/net v0.13.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
-golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
-golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
+google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gvisor.dev/gvisor v0.0.0-20231222013827-149350e5c428 h1:UvBO2UZXf0d1zWJsfD8Robnxa2lyGm8Vnb+Nou5b1no=
+gvisor.dev/gvisor v0.0.0-20231222013827-149350e5c428/go.mod h1:10sU+Uh5KKNv1+2x2A0Gvzt8FjD3ASIhorV3YsauXhk=

info/version.go

@@ -5,82 +5,150 @@ import (
 	"fmt"
 	"os"
 	"runtime"
+	"runtime/debug"
 	"strings"
+	"sync"
 )
 
 var (
-	name         = "[NAME]"
-	version      = "[version unknown]"
-	commit       = "[commit unknown]"
-	license      = "[license unknown]"
-	buildOptions = "[options unknown]"
-	buildUser    = "[user unknown]"
-	buildHost    = "[host unknown]"
-	buildDate    = "[date unknown]"
-	buildSource  = "[source unknown]"
+	name    string
+	license string
 
-	compareVersion bool
+	version       = "dev build"
+	versionNumber = "0.0.0"
+	buildSource   = "unknown"
+	buildTime     = "unknown"
+
+	info     *Info
+	loadInfo sync.Once
 )
 
+func init() {
+	// Replace space placeholders.
+	buildSource = strings.ReplaceAll(buildSource, "_", " ")
+	buildTime = strings.ReplaceAll(buildTime, "_", " ")
+
+	// Convert version string from git tag to expected format.
+	version = strings.TrimSpace(strings.ReplaceAll(strings.TrimPrefix(version, "v"), "_", " "))
+	versionNumber = strings.TrimSpace(strings.TrimSuffix(version, "dev build"))
+	if versionNumber == "" {
+		versionNumber = "0.0.0"
+	}
+
+	// Get build info.
+	buildInfo, _ := debug.ReadBuildInfo()
+	buildSettings := make(map[string]string)
+	for _, setting := range buildInfo.Settings {
+		buildSettings[setting.Key] = setting.Value
+	}
+
+	// Add "dev build" to version if repo is dirty.
+	if buildSettings["vcs.modified"] == "true" &&
+		!strings.HasSuffix(version, "dev build") {
+		version += " dev build"
+	}
+}
+
 // Info holds the programs meta information.
-type Info struct {
-	Name         string
-	Version      string
-	License      string
-	Commit       string
-	BuildOptions string
-	BuildUser    string
-	BuildHost    string
-	BuildDate    string
-	BuildSource  string
+type Info struct { //nolint:maligned
+	Name          string
+	Version       string
+	VersionNumber string
+	License       string
+
+	Source    string
+	BuildTime string
+	CGO       bool
+
+	Commit     string
+	CommitTime string
+	Dirty      bool
+
+	debug.BuildInfo
 }
 
 // Set sets meta information via the main routine. This should be the first thing your program calls.
-func Set(setName string, setVersion string, setLicenseName string, compareVersionToTag bool) {
+func Set(setName string, setVersion string, setLicenseName string) {
 	name = setName
-	version = setVersion
 	license = setLicenseName
-	compareVersion = compareVersionToTag
+
+	if setVersion != "" {
+		version = setVersion
+	}
 }
 
 // GetInfo returns all the meta information about the program.
 func GetInfo() *Info {
-	return &Info{
-		Name:         name,
-		Version:      version,
-		Commit:       commit,
-		License:      license,
-		BuildOptions: buildOptions,
-		BuildUser:    buildUser,
-		BuildHost:    buildHost,
-		BuildDate:    buildDate,
-		BuildSource:  buildSource,
-	}
+	loadInfo.Do(func() {
+		buildInfo, _ := debug.ReadBuildInfo()
+		buildSettings := make(map[string]string)
+		for _, setting := range buildInfo.Settings {
+			buildSettings[setting.Key] = setting.Value
+		}
+
+		info = &Info{
+			Name:          name,
+			Version:       version,
+			VersionNumber: versionNumber,
+			License:       license,
+			Source:        buildSource,
+			BuildTime:     buildTime,
+			CGO:           buildSettings["CGO_ENABLED"] == "1",
+			Commit:        buildSettings["vcs.revision"],
+			CommitTime:    buildSettings["vcs.time"],
+			Dirty:         buildSettings["vcs.modified"] == "true",
+			BuildInfo:     *buildInfo,
+		}
+
+		if info.Commit == "" {
+			info.Commit = "unknown"
+		}
+		if info.CommitTime == "" {
+			info.CommitTime = "unknown"
+		}
+	})
+
+	return info
 }
 
-// Version returns the short version string.
+// Version returns the annotated version.
 func Version() string {
-	if !compareVersion || strings.HasPrefix(commit, fmt.Sprintf("tags/v%s-0-", version)) {
-		return version
-	}
-	return version + "*"
+	return version
+}
+
+// VersionNumber returns the version number only.
+func VersionNumber() string {
+	return versionNumber
 }
 
 // FullVersion returns the full and detailed version string.
 func FullVersion() string {
-	s := ""
-	if !compareVersion || strings.HasPrefix(commit, fmt.Sprintf("tags/v%s-0-", version)) {
-		s += fmt.Sprintf("%s\nversion %s\n", name, version)
-	} else {
-		s += fmt.Sprintf("%s\ndevelopment build, built on top version %s\n", name, version)
+	info := GetInfo()
+	builder := new(strings.Builder)
+
+	// Name and version.
+	builder.WriteString(fmt.Sprintf("%s %s\n", info.Name, version))
+
+	// Build info.
+	cgoInfo := "-cgo"
+	if info.CGO {
+		cgoInfo = "+cgo"
 	}
-	s += fmt.Sprintf("\ncommit %s\n", commit)
-	s += fmt.Sprintf("built with %s (%s) %s/%s\n", runtime.Version(), runtime.Compiler, runtime.GOOS, runtime.GOARCH)
-	s += fmt.Sprintf("  using options %s\n", strings.ReplaceAll(buildOptions, "§", " "))
-	s += fmt.Sprintf("  by %s@%s\n", buildUser, buildHost)
-	s += fmt.Sprintf("  on %s\n", buildDate)
-	s += fmt.Sprintf("\nLicensed under the %s license.\nThe source code is available here: %s", license, buildSource)
-	return s
+	builder.WriteString(fmt.Sprintf("\nbuilt with %s (%s %s) for %s/%s\n", runtime.Version(), runtime.Compiler, cgoInfo, runtime.GOOS, runtime.GOARCH))
+	builder.WriteString(fmt.Sprintf("  at %s\n", info.BuildTime))
+
+	// Commit info.
+	dirtyInfo := "clean"
+	if info.Dirty {
+		dirtyInfo = "dirty"
+	}
+	builder.WriteString(fmt.Sprintf("\ncommit %s (%s)\n", info.Commit, dirtyInfo))
+	builder.WriteString(fmt.Sprintf("  at %s\n", info.CommitTime))
+	builder.WriteString(fmt.Sprintf("  from %s\n", info.Source))
+
+	builder.WriteString(fmt.Sprintf("\nLicensed under the %s license.", license))
+
+	return builder.String()
 }
 
 // CheckVersion checks if the metadata is ok.
@@ -92,19 +160,9 @@ func CheckVersion() error {
 		return nil // testing on windows
 	default:
 		// check version information
-		if name == "[NAME]" {
+		if name == "" || license == "" {
 			return errors.New("must call SetInfo() before calling CheckVersion()")
 		}
-		if version == "[version unknown]" ||
-			commit == "[commit unknown]" ||
-			license == "[license unknown]" ||
-			buildOptions == "[options unknown]" ||
-			buildUser == "[user unknown]" ||
-			buildHost == "[host unknown]" ||
-			buildDate == "[date unknown]" ||
-			buildSource == "[source unknown]" {
-			return errors.New("please build using the supplied build script.\n$ ./build {main.go|...}")
-		}
 	}
 
 	return nil

metrics/api.go

@@ -3,7 +3,6 @@ package metrics
 import (
 	"bytes"
 	"context"
-	"encoding/json"
 	"fmt"
 	"io"
 	"net/http"
@@ -17,20 +16,41 @@ import (
 func registerAPI() error {
 	api.RegisterHandler("/metrics", &metricsAPI{})
 
-	return api.RegisterEndpoint(api.Endpoint{
-		Path:      "metrics/list",
-		Read:      api.PermitAnyone,
-		MimeType:  api.MimeTypeJSON,
-		BelongsTo: module,
-		DataFunc: func(*api.Request) ([]byte, error) {
-			registryLock.RLock()
-			defer registryLock.RUnlock()
-
-			return json.Marshal(registry)
-		},
+	if err := api.RegisterEndpoint(api.Endpoint{
 		Name:        "Export Registered Metrics",
 		Description: "List all registered metrics with their metadata.",
-	})
+		Path:        "metrics/list",
+		Read:        api.Dynamic,
+		BelongsTo:   module,
+		StructFunc: func(ar *api.Request) (any, error) {
+			return ExportMetrics(ar.AuthToken.Read), nil
+		},
+	}); err != nil {
+		return err
+	}
+
+	if err := api.RegisterEndpoint(api.Endpoint{
+		Name:        "Export Metric Values",
+		Description: "List all exportable metric values.",
+		Path:        "metrics/values",
+		Read:        api.Dynamic,
+		Parameters: []api.Parameter{{
+			Method:      http.MethodGet,
+			Field:       "internal-only",
+			Description: "Specify to only return metrics with an alternative internal ID.",
+		}},
+		BelongsTo: module,
+		StructFunc: func(ar *api.Request) (any, error) {
+			return ExportValues(
+				ar.AuthToken.Read,
+				ar.Request.URL.Query().Has("internal-only"),
+			), nil
+		},
+	}); err != nil {
+		return err
+	}
+
+	return nil
 }
 
 type metricsAPI struct{}

metrics/metric.go

@@ -43,6 +43,10 @@ type Options struct {
 	// Name defines an optional human readable name for the metric.
 	Name string
 
+	// InternalID specifies an alternative internal ID that will be used when
+	// exposing the metric via the API in a structured format.
+	InternalID string
+
 	// AlertLimit defines an upper limit that triggers an alert.
 	AlertLimit float64
 

metrics/metric_counter.go

@@ -42,3 +42,8 @@ func NewCounter(id string, labels map[string]string, opts *Options) (*Counter, e
 
 	return m, nil
 }
+
+// CurrentValue returns the current counter value.
+func (c *Counter) CurrentValue() uint64 {
+	return c.Get()
+}

metrics/metric_counter_fetching.go

@@ -50,6 +50,11 @@ func NewFetchingCounter(id string, labels map[string]string, fn func() uint64, o
 	return m, nil
 }
 
+// CurrentValue returns the current counter value.
+func (fc *FetchingCounter) CurrentValue() uint64 {
+	return fc.fetchCnt()
+}
+
 // WritePrometheus writes the metric in the prometheus format to the given writer.
 func (fc *FetchingCounter) WritePrometheus(w io.Writer) {
 	fc.counter.Set(fc.fetchCnt())

metrics/metric_export.go

@@ -0,0 +1,89 @@
+package metrics
+
+import (
+	"github.com/safing/portbase/api"
+)
+
+// UIntMetric is an interface for special functions of uint metrics.
+type UIntMetric interface {
+	CurrentValue() uint64
+}
+
+// FloatMetric is an interface for special functions of float metrics.
+type FloatMetric interface {
+	CurrentValue() float64
+}
+
+// MetricExport is used to export a metric and its current value.
+type MetricExport struct {
+	Metric
+	CurrentValue any
+}
+
+// ExportMetrics exports all registered metrics.
+func ExportMetrics(requestPermission api.Permission) []*MetricExport {
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	export := make([]*MetricExport, 0, len(registry))
+	for _, metric := range registry {
+		// Check permission.
+		if requestPermission < metric.Opts().Permission {
+			continue
+		}
+
+		// Add metric with current value.
+		export = append(export, &MetricExport{
+			Metric:       metric,
+			CurrentValue: getCurrentValue(metric),
+		})
+	}
+
+	return export
+}
+
+// ExportValues exports the values of all supported metrics.
+func ExportValues(requestPermission api.Permission, internalOnly bool) map[string]any {
+	registryLock.RLock()
+	defer registryLock.RUnlock()
+
+	export := make(map[string]any, len(registry))
+	for _, metric := range registry {
+		// Check permission.
+		if requestPermission < metric.Opts().Permission {
+			continue
+		}
+
+		// Get Value.
+		v := getCurrentValue(metric)
+		if v == nil {
+			continue
+		}
+
+		// Get ID.
+		var id string
+		switch {
+		case metric.Opts().InternalID != "":
+			id = metric.Opts().InternalID
+		case internalOnly:
+			continue
+		default:
+			id = metric.LabeledID()
+		}
+
+		// Add to export
+		export[id] = v
+	}
+
+	return export
+}
+
+func getCurrentValue(metric Metric) any {
+	if m, ok := metric.(UIntMetric); ok {
+		return m.CurrentValue()
+	}
+	if m, ok := metric.(FloatMetric); ok {
+		return m.CurrentValue()
+	}
+	return nil
+}

metrics/metric_gauge.go

@@ -39,3 +39,8 @@ func NewGauge(id string, labels map[string]string, fn func() float64, opts *Opti
 
 	return m, nil
 }
+
+// CurrentValue returns the current gauge value.
+func (g *Gauge) CurrentValue() float64 {
+	return g.Get()
+}

metrics/metrics_host.go

@@ -16,7 +16,7 @@ import (
 
 const hostStatTTL = 1 * time.Second
 
-func registeHostMetrics() (err error) {
+func registerHostMetrics() (err error) {
 	// Register load average metrics.
 	_, err = NewGauge("host/load/avg/1", nil, getFloat64HostStat(LoadAvg1), &Options{Name: "Host Load Avg 1min", Permission: api.PermitUser})
 	if err != nil {
@@ -127,7 +127,7 @@ func LoadAvg5() (loadAvg float64, ok bool) {
 	return 0, false
 }
 
-// LoadAvg15 returns the  5-minute average system load.
+// LoadAvg15 returns the 15-minute average system load.
 func LoadAvg15() (loadAvg float64, ok bool) {
 	if stat := getLoadAvg(); stat != nil {
 		return stat.Load15 / float64(runtime.NumCPU()), true

metrics/metrics_info.go

@@ -3,29 +3,33 @@ package metrics
 import (
 	"runtime"
 	"strings"
+	"sync/atomic"
 
 	"github.com/safing/portbase/info"
 )
 
+var reportedStart atomic.Bool
+
 func registerInfoMetric() error {
 	meta := info.GetInfo()
 	_, err := NewGauge(
 		"info",
 		map[string]string{
-			"version":       checkUnknown(meta.Version),
-			"commit":        checkUnknown(meta.Commit),
-			"build_options": checkUnknown(meta.BuildOptions),
-			"build_user":    checkUnknown(meta.BuildUser),
-			"build_host":    checkUnknown(meta.BuildHost),
-			"build_date":    checkUnknown(meta.BuildDate),
-			"build_source":  checkUnknown(meta.BuildSource),
-			"go_os":         runtime.GOOS,
-			"go_arch":       runtime.GOARCH,
-			"go_version":    runtime.Version(),
-			"go_compiler":   runtime.Compiler,
-			"comment":       commentOption(),
+			"version":      checkUnknown(meta.Version),
+			"commit":       checkUnknown(meta.Commit),
+			"build_date":   checkUnknown(meta.BuildTime),
+			"build_source": checkUnknown(meta.Source),
+			"go_os":        runtime.GOOS,
+			"go_arch":      runtime.GOARCH,
+			"go_version":   runtime.Version(),
+			"go_compiler":  runtime.Compiler,
+			"comment":      commentOption(),
 		},
 		func() float64 {
+			// Report as 0 the first time in order to detect (re)starts.
+			if reportedStart.CompareAndSwap(false, true) {
+				return 0
+			}
 			return 1
 		},
 		nil,

metrics/metrics_logs.go

@@ -5,7 +5,7 @@ import (
 	"github.com/safing/portbase/log"
 )
 
-func registeLogMetrics() (err error) {
+func registerLogMetrics() (err error) {
 	_, err = NewFetchingCounter(
 		"logs/warning/total",
 		nil,

metrics/module.go

@@ -58,11 +58,11 @@ func start() error {
 		return err
 	}
 
-	if err := registeHostMetrics(); err != nil {
+	if err := registerHostMetrics(); err != nil {
 		return err
 	}
 
-	if err := registeLogMetrics(); err != nil {
+	if err := registerLogMetrics(); err != nil {
 		return err
 	}
 
@@ -78,6 +78,17 @@ func start() error {
 }
 
 func stop() error {
+	// Wait until the metrics pusher is done, as it may have started reporting
+	// and may report a higher number than we store to disk. For persistent
+	// metrics it can then happen that the first report is lower than the
+	// previous report, making prometheus think that all that happened since the
+	// last report, due to the automatic restart detection.
+
+	// The registry is read locked when writing metrics.
+	// Write lock the registry to make sure all writes are finished.
+	registryLock.Lock()
+	registryLock.Unlock() //nolint:staticcheck
+
 	storePersistentMetrics()
 
 	return nil
@@ -92,6 +103,10 @@ func register(m Metric) error {
 		if m.LabeledID() == registeredMetric.LabeledID() {
 			return ErrAlreadyRegistered
 		}
+		if m.Opts().InternalID != "" &&
+			m.Opts().InternalID == registeredMetric.Opts().InternalID {
+			return fmt.Errorf("%w with this internal ID", ErrAlreadyRegistered)
+		}
 	}
 
 	// Add new metric to registry and sort it.
@@ -101,6 +116,10 @@ func register(m Metric) error {
 	// Set flag that first metric is now registered.
 	firstMetricRegistered = true
 
+	if module.Status() < modules.StatusStarting {
+		return fmt.Errorf("registering metric %q too early", m.ID())
+	}
+
 	return nil
 }
 

metrics/persistence.go

@@ -25,7 +25,7 @@ var (
 	})
 
 	// ErrAlreadyInitialized is returned when trying to initialize an option
-	// more than once.
+	// more than once or if the time window for initializing is over.
 	ErrAlreadyInitialized = errors.New("already initialized")
 )
 
@@ -55,7 +55,7 @@ func EnableMetricPersistence(key string) error {
 
 	// Load metrics from storage.
 	var err error
-	storage, err = getMetricsStorage(key)
+	storage, err = getMetricsStorage(storageKey)
 	switch {
 	case err == nil:
 		// Continue.

modules/events.go

@@ -104,7 +104,7 @@ func (m *Module) InjectEvent(sourceEventName, targetModuleName, targetEventName
 func (m *Module) runEventHook(hook *eventHook, event string, data interface{}) {
 	// check if source module is ready for handling
 	if m.Status() != StatusOnline {
-		// target module has not yet fully started, wait until start is complete
+		// source module has not yet fully started, wait until start is complete
 		select {
 		case <-m.StartCompleted():
 			// continue with hook execution

modules/modules.go

@@ -57,10 +57,11 @@ type Module struct { //nolint:maligned
 	// start
 	startComplete chan struct{}
 	// stop
-	Ctx          context.Context
-	cancelCtx    func()
-	stopFlag     *abool.AtomicBool
-	stopComplete chan struct{}
+	Ctx           context.Context
+	cancelCtx     func()
+	stopFlag      *abool.AtomicBool
+	stopCompleted *abool.AtomicBool
+	stopComplete  chan struct{}
 
 	// workers/tasks
 	ctrlFuncRunning *abool.AtomicBool
@@ -255,12 +256,10 @@ func (m *Module) checkIfStopComplete() {
 		atomic.LoadInt32(m.taskCnt) == 0 &&
 		atomic.LoadInt32(m.microTaskCnt) == 0 {
 
-		m.Lock()
-		defer m.Unlock()
-
-		if m.stopComplete != nil {
+		if m.stopCompleted.SetToIf(false, true) {
+			m.Lock()
+			defer m.Unlock()
 			close(m.stopComplete)
-			m.stopComplete = nil
 		}
 	}
 }
@@ -283,60 +282,56 @@ func (m *Module) stop(reports chan *report) {
 	// Reset start/stop signal channels.
 	m.startComplete = make(chan struct{})
 	m.stopComplete = make(chan struct{})
+	m.stopCompleted.SetTo(false)
 
-	// Make a copy of the stop channel.
-	stopComplete := m.stopComplete
-
-	// Set status and cancel context.
+	// Set status.
 	m.status = StatusStopping
-	m.stopFlag.Set()
-	m.cancelCtx()
 
-	go m.stopAllTasks(reports, stopComplete)
+	go m.stopAllTasks(reports)
 }
 
-func (m *Module) stopAllTasks(reports chan *report, stopComplete chan struct{}) {
-	// start shutdown function
-	var stopFnError error
-	stopFuncRunning := abool.New()
-	if m.stopFn != nil {
-		stopFuncRunning.Set()
-		go func() {
-			stopFnError = m.runCtrlFn("stop module", m.stopFn)
-			stopFuncRunning.UnSet()
-			m.checkIfStopComplete()
-		}()
-	} else {
-		m.checkIfStopComplete()
-	}
+func (m *Module) stopAllTasks(reports chan *report) {
+	// Manually set the control function flag in order to stop completion by race
+	// condition before stop function has even started.
+	m.ctrlFuncRunning.Set()
+
+	// Set stop flag for everyone checking this flag before we activate any stop trigger.
+	m.stopFlag.Set()
+
+	// Cancel the context to notify all workers and tasks.
+	m.cancelCtx()
+
+	// Start stop function.
+	stopFnError := m.startCtrlFn("stop module", m.stopFn)
 
 	// wait for results
 	select {
-	case <-stopComplete:
-	// case <-time.After(moduleStopTimeout):
+	case <-m.stopComplete:
+		// Complete!
 	case <-time.After(moduleStopTimeout):
 		log.Warningf(
-			"%s: timed out while waiting for stopfn/workers/tasks to finish: stopFn=%v/%v workers=%d tasks=%d microtasks=%d, continuing shutdown...",
+			"%s: timed out while waiting for stopfn/workers/tasks to finish: stopFn=%v workers=%d tasks=%d microtasks=%d, continuing shutdown...",
 			m.Name,
-			stopFuncRunning.IsSet(), m.ctrlFuncRunning.IsSet(),
+			m.ctrlFuncRunning.IsSet(),
 			atomic.LoadInt32(m.workerCnt),
 			atomic.LoadInt32(m.taskCnt),
 			atomic.LoadInt32(m.microTaskCnt),
 		)
 	}
 
-	// collect error
+	// Check for stop fn status.
 	var err error
-	if stopFuncRunning.IsNotSet() && stopFnError != nil {
-		err = stopFnError
-	}
-	// set status
-	if err != nil {
-		m.Error(
-			fmt.Sprintf("%s:stop-failed", m.Name),
-			fmt.Sprintf("Stopping module %s failed", m.Name),
-			fmt.Sprintf("Failed to stop module: %s", err.Error()),
-		)
+	select {
+	case err = <-stopFnError:
+		if err != nil {
+			// Set error as module error.
+			m.Error(
+				fmt.Sprintf("%s:stop-failed", m.Name),
+				fmt.Sprintf("Stopping module %s failed", m.Name),
+				fmt.Sprintf("Failed to stop module: %s", err.Error()),
+			)
+		}
+	default:
 	}
 
 	// Always set to offline in order to let other modules shutdown in order.
@@ -384,7 +379,7 @@ func initNewModule(name string, prep, start, stop func() error, dependencies ...
 		Name:                name,
 		enabled:             abool.NewBool(false),
 		enabledAsDependency: abool.NewBool(false),
-		sleepMode:           abool.NewBool(true),
+		sleepMode:           abool.NewBool(true), // Change (for init) is triggered below.
 		sleepWaitingChannel: make(chan time.Time),
 		prepFn:              prep,
 		startFn:             start,
@@ -393,6 +388,7 @@ func initNewModule(name string, prep, start, stop func() error, dependencies ...
 		Ctx:                 ctx,
 		cancelCtx:           cancelCtx,
 		stopFlag:            abool.NewBool(false),
+		stopCompleted:       abool.NewBool(true),
 		ctrlFuncRunning:     abool.NewBool(false),
 		workerCnt:           &workerCnt,
 		taskCnt:             &taskCnt,
@@ -401,7 +397,7 @@ func initNewModule(name string, prep, start, stop func() error, dependencies ...
 		depNames:            dependencies,
 	}
 
-	// Sleep mode is disabled by default
+	// Sleep mode is disabled by default.
 	newModule.Sleep(false)
 
 	return newModule

modules/start.go

@@ -24,6 +24,11 @@ func SetGlobalPrepFn(fn func() error) {
 	}
 }
 
+// IsStarting returns whether the initial global start is still in progress.
+func IsStarting() bool {
+	return !initialStartCompleted.IsSet()
+}
+
 // Start starts all modules in the correct order. In case of an error, it will automatically shutdown again.
 func Start() error {
 	if !modulesLocked.SetToIf(false, true) {

modules/worker.go

@@ -53,6 +53,7 @@ func (m *Module) RunWorker(name string, fn func(context.Context) error) error {
 }
 
 // StartServiceWorker starts a generic worker, which is automatically restarted in case of an error. A call to StartServiceWorker runs the service-worker in a new goroutine and returns immediately. `backoffDuration` specifies how to long to wait before restarts, multiplied by the number of failed attempts. Pass `0` for the default backoff duration. For custom error remediation functionality, build your own error handling procedure using calls to RunWorker.
+// Returning nil error or context.Canceled will stop the service worker.
 func (m *Module) StartServiceWorker(name string, backoffDuration time.Duration, fn func(context.Context) error) {
 	if m == nil {
 		log.Errorf(`modules: cannot start service worker "%s" with nil module`, name)
@@ -81,34 +82,36 @@ func (m *Module) runServiceWorker(name string, backoffDuration time.Duration, fn
 		}
 
 		err := m.runWorker(name, fn)
-		if err != nil {
-			if !errors.Is(err, ErrRestartNow) {
-				// reset fail counter if running without error for some time
-				if time.Now().Add(-5 * time.Minute).After(lastFail) {
-					failCnt = 0
-				}
-				// increase fail counter and set last failed time
-				failCnt++
-				lastFail = time.Now()
-				// log error
-				sleepFor := time.Duration(failCnt) * backoffDuration
-				if errors.Is(err, context.Canceled) {
-					log.Debugf("%s: service-worker %s was canceled (%d): %s - restarting in %s", m.Name, name, failCnt, err, sleepFor)
-				} else {
-					log.Errorf("%s: service-worker %s failed (%d): %s - restarting in %s", m.Name, name, failCnt, err, sleepFor)
-				}
-				select {
-				case <-time.After(sleepFor):
-				case <-m.Ctx.Done():
-					return
-				}
-				// loop to restart
-			} else {
-				log.Infof("%s: service-worker %s %s - restarting now", m.Name, name, err)
-			}
-		} else {
-			// finish
+		switch {
+		case err == nil:
+			// No error means that the worker is finished.
 			return
+
+		case errors.Is(err, context.Canceled):
+			// A canceled context also means that the worker is finished.
+			return
+
+		case errors.Is(err, ErrRestartNow):
+			// Worker requested a restart - silently continue with loop.
+
+		default:
+			// Any other errors triggers a restart with backoff.
+
+			// Reset fail counter if running without error for some time.
+			if time.Now().Add(-5 * time.Minute).After(lastFail) {
+				failCnt = 0
+			}
+			// Increase fail counter and set last failed time.
+			failCnt++
+			lastFail = time.Now()
+			// Log error and back off for some time.
+			sleepFor := time.Duration(failCnt) * backoffDuration
+			log.Errorf("%s: service-worker %s failed (%d): %s - restarting in %s", m.Name, name, failCnt, err, sleepFor)
+			select {
+			case <-time.After(sleepFor):
+			case <-m.Ctx.Done():
+				return
+			}
 		}
 	}
 }
@@ -132,10 +135,7 @@ func (m *Module) runWorker(name string, fn func(context.Context) error) (err err
 }
 
 func (m *Module) runCtrlFnWithTimeout(name string, timeout time.Duration, fn func() error) error {
-	stopFnError := make(chan error)
-	go func() {
-		stopFnError <- m.runCtrlFn(name, fn)
-	}()
+	stopFnError := m.startCtrlFn(name, fn)
 
 	// wait for results
 	select {
@@ -146,26 +146,44 @@ func (m *Module) runCtrlFnWithTimeout(name string, timeout time.Duration, fn fun
 	}
 }
 
-func (m *Module) runCtrlFn(name string, fn func() error) (err error) {
+func (m *Module) startCtrlFn(name string, fn func() error) chan error {
+	ctrlFnError := make(chan error, 1)
+
+	// If no function is given, still act as if it was run.
 	if fn == nil {
-		return
+		// Signal finish.
+		m.ctrlFuncRunning.UnSet()
+		m.checkIfStopComplete()
+
+		// Report nil error and return.
+		ctrlFnError <- nil
+		return ctrlFnError
 	}
 
-	if m.ctrlFuncRunning.SetToIf(false, true) {
-		defer m.ctrlFuncRunning.SetToIf(true, false)
-	}
+	// Signal that a control function is running.
+	m.ctrlFuncRunning.Set()
 
-	defer func() {
-		// recover from panic
-		panicVal := recover()
-		if panicVal != nil {
-			me := m.NewPanicError(name, "module-control", panicVal)
-			me.Report()
-			err = me
-		}
+	// Start control function in goroutine.
+	go func() {
+		// Recover from panic and reset control function signal.
+		defer func() {
+			// recover from panic
+			panicVal := recover()
+			if panicVal != nil {
+				me := m.NewPanicError(name, "module-control", panicVal)
+				me.Report()
+				ctrlFnError <- fmt.Errorf("panic: %s", panicVal)
+			}
+
+			// Signal finish.
+			m.ctrlFuncRunning.UnSet()
+			m.checkIfStopComplete()
+		}()
+
+		// Run control function and report error.
+		err := fn()
+		ctrlFnError <- err
 	}()
 
-	// run
-	err = fn()
-	return
+	return ctrlFnError
 }

notifications/notification.go

@@ -393,6 +393,17 @@ func (n *Notification) Update(expires int64) {
 
 // Delete (prematurely) cancels and deletes a notification.
 func (n *Notification) Delete() {
+	// Dismiss notification.
+	func() {
+		n.lock.Lock()
+		defer n.lock.Unlock()
+
+		if n.actionTrigger != nil {
+			close(n.actionTrigger)
+			n.actionTrigger = nil
+		}
+	}()
+
 	n.delete(true)
 }
 

portbase.go

@@ -10,7 +10,7 @@ import (
 
 func main() {
 	// Set Info
-	info.Set("Portbase", "0.0.1", "GPLv3", false)
+	info.Set("Portbase", "0.0.1", "GPLv3")
 
 	// Run
 	os.Exit(run.Run())

updater/registry.go

@@ -43,8 +43,12 @@ type ResourceRegistry struct {
 	// version. Even if false, a pre-release version will still be used if it is
 	// defined as the current version by an index.
 	UsePreReleases bool
-	DevMode        bool
-	Online         bool
+
+	// DevMode specifies if a local 0.0.0 version should be always chosen, when available.
+	DevMode bool
+
+	// Online specifies if resources may be downloaded if not available locally.
+	Online bool
 
 	// StateNotifyFunc may be set to receive any changes to the registry state.
 	// The specified function may lock the state, but may not block or take a

updater/resource.go

@@ -112,7 +112,26 @@ func (rv *ResourceVersion) EqualsVersion(version string) bool {
 // A version is selectable if it's not blacklisted and either already locally
 // available or ready to be downloaded.
 func (rv *ResourceVersion) isSelectable() bool {
-	return !rv.Blacklisted && (rv.Available || rv.resource.registry.Online)
+	switch {
+	case rv.Blacklisted:
+		// Should not be used.
+		return false
+	case rv.Available:
+		// Is available locally, use!
+		return true
+	case !rv.resource.registry.Online:
+		// Cannot download, because registry is set to offline.
+		return false
+	case rv.resource.Index == nil:
+		// Cannot download, because resource is not part of an index.
+		return false
+	case !rv.resource.Index.AutoDownload:
+		// Cannot download, because index may not automatically download.
+		return false
+	default:
+		// Is not available locally, but we are allowed to download it on request!
+		return true
+	}
 }
 
 // isBetaVersionNumber checks if rv is marked as a beta version by checking
@@ -290,8 +309,13 @@ func (res *Resource) selectVersion() {
 	sort.Sort(res)
 
 	// export after we finish
+	var fallback bool
 	defer func() {
-		log.Tracef("updater: selected version %s for resource %s", res.SelectedVersion, res.Identifier)
+		if fallback {
+			log.Tracef("updater: selected version %s (as fallback) for resource %s", res.SelectedVersion, res.Identifier)
+		} else {
+			log.Debugf("updater: selected version %s for resource %s", res.SelectedVersion, res.Identifier)
+		}
 
 		if res.inUse() &&
 			res.SelectedVersion != res.ActiveVersion && // new selected version does not match previously selected version
@@ -356,7 +380,7 @@ func (res *Resource) selectVersion() {
 
 	// 5) Default to newest.
 	res.SelectedVersion = res.Versions[0]
-	log.Warningf("updater: falling back to version %s for %s because we failed to find a selectable one", res.SelectedVersion, res.Identifier)
+	fallback = true
 }
 
 // Blacklist blacklists the specified version and selects a new version.

updater/resource_test.go

@@ -45,6 +45,8 @@ func TestVersionSelection(t *testing.T) {
 	registry.UsePreReleases = true
 	registry.DevMode = true
 	registry.Online = true
+	res.Index = &Index{AutoDownload: true}
+
 	res.selectVersion()
 	if res.SelectedVersion.VersionNumber != "0.0.0" {
 		t.Errorf("selected version should be 0.0.0, not %s", res.SelectedVersion.VersionNumber)

updater/updating.go

@@ -47,7 +47,7 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
 	// Get pending resources and update status.
 	pendingResourceVersions, _ := reg.GetPendingDownloads(true, false)
 	reg.state.ReportUpdateCheck(
-		identifiersFromResourceVersions(pendingResourceVersions),
+		humanInfoFromResourceVersions(pendingResourceVersions),
 		nil,
 	)
 
@@ -183,14 +183,14 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
 }
 
 // DownloadUpdates checks if updates are available and downloads updates of used components.
-func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context, automaticOnly bool) error {
+func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context, includeManual bool) error {
 	// Start registry operation.
 	reg.state.StartOperation(StateDownloading)
 	defer reg.state.EndOperation()
 
 	// Get pending updates.
-	toUpdate, missingSigs := reg.GetPendingDownloads(!automaticOnly, true)
-	downloadDetailsResources := identifiersFromResourceVersions(toUpdate)
+	toUpdate, missingSigs := reg.GetPendingDownloads(includeManual, true)
+	downloadDetailsResources := humanInfoFromResourceVersions(toUpdate)
 	reg.state.UpdateOperationDetails(&StateDownloadingDetails{
 		Resources: downloadDetailsResources,
 	})
@@ -348,11 +348,11 @@ func (reg *ResourceRegistry) GetPendingDownloads(manual, auto bool) (resources,
 	return toUpdate, missingSigs
 }
 
-func identifiersFromResourceVersions(resourceVersions []*ResourceVersion) []string {
+func humanInfoFromResourceVersions(resourceVersions []*ResourceVersion) []string {
 	identifiers := make([]string, len(resourceVersions))
 
 	for i, rv := range resourceVersions {
-		identifiers[i] = rv.resource.Identifier
+		identifiers[i] = fmt.Sprintf("%s v%s", rv.resource.Identifier, rv.VersionNumber)
 	}
 
 	return identifiers

utils/call_limiter.go

@@ -0,0 +1,87 @@
+package utils
+
+import (
+	"sync"
+	"sync/atomic"
+	"time"
+)
+
+// CallLimiter bundles concurrent calls and optionally limits how fast a function is called.
+type CallLimiter struct {
+	pause time.Duration
+
+	inLock   sync.Mutex
+	lastExec time.Time
+
+	waiters atomic.Int32
+	outLock sync.Mutex
+}
+
+// NewCallLimiter returns a new call limiter.
+// Set minPause to zero to disable the minimum pause between calls.
+func NewCallLimiter(minPause time.Duration) *CallLimiter {
+	return &CallLimiter{
+		pause: minPause,
+	}
+}
+
+// Do executes the given function.
+// All concurrent calls to Do are bundled and return when f() finishes.
+// Waits until the minimum pause is over before executing f() again.
+func (l *CallLimiter) Do(f func()) {
+	// Wait for the previous waiters to exit.
+	l.inLock.Lock()
+
+	// Defer final unlock to safeguard from panics.
+	defer func() {
+		// Execution is finished - leave.
+		// If we are the last waiter, let the next batch in.
+		if l.waiters.Add(-1) == 0 {
+			l.inLock.Unlock()
+		}
+	}()
+
+	// Check if we are the first waiter.
+	if l.waiters.Add(1) == 1 {
+		// Take the lead on this execution run.
+		l.lead(f)
+	} else {
+		// We are not the first waiter, let others in.
+		l.inLock.Unlock()
+	}
+
+	// Wait for execution to complete.
+	l.outLock.Lock()
+	l.outLock.Unlock() //nolint:staticcheck
+
+	// Last statement is in defer above.
+}
+
+func (l *CallLimiter) lead(f func()) {
+	// Make all others wait while we execute the function.
+	l.outLock.Lock()
+
+	// Unlock in lock until execution is finished.
+	l.inLock.Unlock()
+
+	// Transition from out lock to in lock when done.
+	defer func() {
+		// Update last execution time.
+		l.lastExec = time.Now().UTC()
+		// Stop newcomers from waiting on previous execution.
+		l.inLock.Lock()
+		// Allow waiters to leave.
+		l.outLock.Unlock()
+	}()
+
+	// Wait for the minimum duration between executions.
+	if l.pause > 0 {
+		sinceLastExec := time.Since(l.lastExec)
+		if sinceLastExec < l.pause {
+			time.Sleep(l.pause - sinceLastExec)
+		}
+	}
+
+	// Execute.
+	f()
+}

utils/call_limiter_test.go

@@ -0,0 +1,91 @@
+package utils
+
+import (
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/tevino/abool"
+)
+
+func TestCallLimiter(t *testing.T) {
+	t.Parallel()
+
+	pause := 10 * time.Millisecond
+	oa := NewCallLimiter(pause)
+	executed := abool.New()
+	var testWg sync.WaitGroup
+
+	// One execution should gobble up the whole batch.
+	// We are doing this without sleep in function, so dummy exec first to trigger first pause.
+	oa.Do(func() {})
+	// Start
+	for i := 0; i < 10; i++ {
+		testWg.Add(100)
+		for i := 0; i < 100; i++ {
+			go func() {
+				oa.Do(func() {
+					if !executed.SetToIf(false, true) {
+						t.Errorf("concurrent execution!")
+					}
+				})
+				testWg.Done()
+			}()
+		}
+		testWg.Wait()
+		// Check if function was executed at least once.
+		if executed.IsNotSet() {
+			t.Errorf("no execution!")
+		}
+		executed.UnSet() // reset check
+	}
+
+	// Wait for pause to reset.
+	time.Sleep(pause)
+
+	// Continuous use with re-execution.
+	// Choose values so that about 10 executions are expected
+	var execs uint32
+	testWg.Add(200)
+	for i := 0; i < 200; i++ {
+		go func() {
+			oa.Do(func() {
+				atomic.AddUint32(&execs, 1)
+				time.Sleep(10 * time.Millisecond)
+			})
+			testWg.Done()
+		}()
+
+		// Start one goroutine every 1ms.
+		time.Sleep(1 * time.Millisecond)
+	}
+
+	testWg.Wait()
+	if execs <= 5 {
+		t.Errorf("unexpected low exec count: %d", execs)
+	}
+	if execs >= 15 {
+		t.Errorf("unexpected high exec count: %d", execs)
+	}
+
+	// Wait for pause to reset.
+	time.Sleep(pause)
+
+	// Check if the limiter correctly handles panics.
+	testWg.Add(100)
+	for i := 0; i < 100; i++ {
+		go func() {
+			defer func() {
+				_ = recover()
+				testWg.Done()
+			}()
+			oa.Do(func() {
+				time.Sleep(1 * time.Millisecond)
+				panic("test")
+			})
+		}()
+		time.Sleep(100 * time.Microsecond)
+	}
+	testWg.Wait()
+}

utils/mimetypes.go

@@ -0,0 +1,78 @@
+package utils
+
+import "strings"
+
+// Do not depend on the OS for mimetypes.
+// A Windows update screwed us over here and broke all the automatic mime
+// typing via Go in April 2021.
+
+// MimeTypeByExtension returns a mimetype for the given file name extension,
+// which must including the leading dot.
+// If the extension is not known, the call returns with ok=false and,
+// additionally, a default "application/octet-stream" mime type is returned.
+func MimeTypeByExtension(ext string) (mimeType string, ok bool) {
+	mimeType, ok = mimeTypes[strings.ToLower(ext)]
+	if ok {
+		return
+	}
+
+	return defaultMimeType, false
+}
+
+var (
+	defaultMimeType = "application/octet-stream"
+
+	mimeTypes = map[string]string{
+		".7z":    "application/x-7z-compressed",
+		".atom":  "application/atom+xml",
+		".css":   "text/css; charset=utf-8",
+		".csv":   "text/csv; charset=utf-8",
+		".deb":   "application/x-debian-package",
+		".epub":  "application/epub+zip",
+		".es":    "application/ecmascript",
+		".flv":   "video/x-flv",
+		".gif":   "image/gif",
+		".gz":    "application/gzip",
+		".htm":   "text/html; charset=utf-8",
+		".html":  "text/html; charset=utf-8",
+		".jpeg":  "image/jpeg",
+		".jpg":   "image/jpeg",
+		".js":    "text/javascript; charset=utf-8",
+		".json":  "application/json; charset=utf-8",
+		".m3u":   "audio/mpegurl",
+		".m4a":   "audio/mpeg",
+		".md":    "text/markdown; charset=utf-8",
+		".mjs":   "text/javascript; charset=utf-8",
+		".mov":   "video/quicktime",
+		".mp3":   "audio/mpeg",
+		".mp4":   "video/mp4",
+		".mpeg":  "video/mpeg",
+		".mpg":   "video/mpeg",
+		".ogg":   "audio/ogg",
+		".ogv":   "video/ogg",
+		".otf":   "font/otf",
+		".pdf":   "application/pdf",
+		".png":   "image/png",
+		".qt":    "video/quicktime",
+		".rar":   "application/rar",
+		".rtf":   "application/rtf",
+		".svg":   "image/svg+xml",
+		".tar":   "application/x-tar",
+		".tiff":  "image/tiff",
+		".ts":    "video/MP2T",
+		".ttc":   "font/collection",
+		".ttf":   "font/ttf",
+		".txt":   "text/plain; charset=utf-8",
+		".wasm":  "application/wasm",
+		".wav":   "audio/x-wav",
+		".webm":  "video/webm",
+		".webp":  "image/webp",
+		".woff":  "font/woff",
+		".woff2": "font/woff2",
+		".xml":   "text/xml; charset=utf-8",
+		".xz":    "application/x-xz",
+		".yaml":  "application/yaml; charset=utf-8",
+		".yml":   "application/yaml; charset=utf-8",
+		".zip":   "application/zip",
+	}
+)

utils/onceagain.go

@@ -7,7 +7,13 @@ import (
 	"sync/atomic"
 )
 
-// OnceAgain is an object that will perform only one action "in flight". It's basically the same as sync.Once, but is automatically reused when the function was executed and everyone who waited has left.
+// OnceAgain is an object that will perform only one action "in flight". It's
+// basically the same as sync.Once, but is automatically reused when the
+// function was executed and everyone who waited has left.
+// Important: This is somewhat racy when used heavily as it only resets _after_
+// everyone who waited has left. So, while some goroutines are waiting to be
+// activated again to leave the waiting state, other goroutines will call Do()
+// without executing the function again.
 type OnceAgain struct {
 	// done indicates whether the action has been performed.
 	// It is first in the struct because it is used in the hot path.

utils/onceagain_test.go

@@ -16,7 +16,7 @@ func TestOnceAgain(t *testing.T) {
 	executed := abool.New()
 	var testWg sync.WaitGroup
 
-	// basic
+	// One execution should gobble up the whole batch.
 	for i := 0; i < 10; i++ {
 		testWg.Add(100)
 		for i := 0; i < 100; i++ {
@@ -34,7 +34,8 @@ func TestOnceAgain(t *testing.T) {
 		executed.UnSet() // reset check
 	}
 
-	// streaming
+	// Continuous use with re-execution.
+	// Choose values so that about 10 executions are expected
 	var execs uint32
 	testWg.Add(100)
 	for i := 0; i < 100; i++ {
@@ -50,7 +51,10 @@ func TestOnceAgain(t *testing.T) {
 	}
 
 	testWg.Wait()
-	if execs >= 20 {
+	if execs <= 8 {
+		t.Errorf("unexpected low exec count: %d", execs)
+	}
+	if execs >= 12 {
 		t.Errorf("unexpected high exec count: %d", execs)
 	}
 }

utils/osdetail/binmeta.go

@@ -1,121 +0,0 @@
-package osdetail
-
-import (
-	"path/filepath"
-	"regexp"
-	"strings"
-)
-
-var (
-	segmentsSplitter  = regexp.MustCompile("[^A-Za-z0-9]*[A-Z]?[a-z0-9]*")
-	nameOnly          = regexp.MustCompile("^[A-Za-z0-9]+$")
-	delimitersAtStart = regexp.MustCompile("^[^A-Za-z0-9]+")
-	delimitersOnly    = regexp.MustCompile("^[^A-Za-z0-9]+$")
-	removeQuotes      = strings.NewReplacer(`"`, ``, `'`, ``)
-)
-
-// GenerateBinaryNameFromPath generates a more human readable binary name from
-// the given path. This function is used as fallback in the GetBinaryName
-// functions.
-func GenerateBinaryNameFromPath(path string) string {
-	// Get file name from path.
-	_, fileName := filepath.Split(path)
-
-	// Split up into segments.
-	segments := segmentsSplitter.FindAllString(fileName, -1)
-
-	// Remove last segment if it's an extension.
-	if len(segments) >= 2 {
-		switch strings.ToLower(segments[len(segments)-1]) {
-		case
-			".exe",      // Windows Executable
-			".msi",      // Windows Installer
-			".bat",      // Windows Batch File
-			".cmd",      // Windows Command Script
-			".ps1",      // Windows Powershell Cmdlet
-			".run",      // Linux Executable
-			".appimage", // Linux AppImage
-			".app",      // MacOS Executable
-			".action",   // MacOS Automator Action
-			".out":      // Generic Compiled Executable
-			segments = segments[:len(segments)-1]
-		}
-	}
-
-	// Debugging snippet:
-	// fmt.Printf("segments: %s\n", segments)
-
-	// Go through segments and collect name parts.
-	nameParts := make([]string, 0, len(segments))
-	var fragments string
-	for _, segment := range segments {
-		// Group very short segments.
-		if len(delimitersAtStart.ReplaceAllString(segment, "")) <= 2 {
-			fragments += segment
-			continue
-		} else if fragments != "" {
-			nameParts = append(nameParts, fragments)
-			fragments = ""
-		}
-
-		// Add segment to name.
-		nameParts = append(nameParts, segment)
-	}
-	// Add last fragment.
-	if fragments != "" {
-		nameParts = append(nameParts, fragments)
-	}
-
-	// Debugging snippet:
-	// fmt.Printf("parts: %s\n", nameParts)
-
-	// Post-process name parts
-	for i := range nameParts {
-		// Remove any leading delimiters.
-		nameParts[i] = delimitersAtStart.ReplaceAllString(nameParts[i], "")
-
-		// Title-case name-only parts.
-		if nameOnly.MatchString(nameParts[i]) {
-			nameParts[i] = strings.Title(nameParts[i]) //nolint:staticcheck
-		}
-	}
-
-	// Debugging snippet:
-	// fmt.Printf("final: %s\n", nameParts)
-
-	return strings.Join(nameParts, " ")
-}
-
-func cleanFileDescription(fileDescr string) string {
-	fields := strings.Fields(fileDescr)
-
-	// Clean out and `"` and `'`.
-	for i := range fields {
-		fields[i] = removeQuotes.Replace(fields[i])
-	}
-
-	// If there is a 1 or 2 character delimiter field, only use fields before it.
-	endIndex := len(fields)
-	for i, field := range fields {
-		// Ignore the first field as well as fields with more than two characters.
-		if i >= 1 && len(field) <= 2 && !nameOnly.MatchString(field) {
-			endIndex = i
-			break
-		}
-	}
-
-	// Concatenate name
-	binName := strings.Join(fields[:endIndex], " ")
-
-	// If there are multiple sentences, only use the first.
-	if strings.Contains(binName, ". ") {
-		binName = strings.SplitN(binName, ". ", 2)[0]
-	}
-
-	// If does not have any characters or numbers, return an empty string.
-	if delimitersOnly.MatchString(binName) {
-		return ""
-	}
-
-	return strings.TrimSpace(binName)
-}

utils/osdetail/binmeta_default.go

@@ -1,15 +0,0 @@
-//go:build !windows
-
-package osdetail
-
-// GetBinaryNameFromSystem queries the operating system for a human readable
-// name for the given binary path.
-func GetBinaryNameFromSystem(path string) (string, error) {
-	return "", ErrNotSupported
-}
-
-// GetBinaryIconFromSystem queries the operating system for the associated icon
-// for a given binary path.
-func GetBinaryIconFromSystem(path string) (string, error) {
-	return "", ErrNotSupported
-}

utils/osdetail/binmeta_test.go

@@ -1,47 +0,0 @@
-package osdetail
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestGenerateBinaryNameFromPath(t *testing.T) {
-	t.Parallel()
-
-	assert.Equal(t, "Nslookup", GenerateBinaryNameFromPath("nslookup.exe"))
-	assert.Equal(t, "System Settings", GenerateBinaryNameFromPath("SystemSettings.exe"))
-	assert.Equal(t, "One Drive Setup", GenerateBinaryNameFromPath("OneDriveSetup.exe"))
-	assert.Equal(t, "Msedge", GenerateBinaryNameFromPath("msedge.exe"))
-	assert.Equal(t, "SIH Client", GenerateBinaryNameFromPath("SIHClient.exe"))
-	assert.Equal(t, "Openvpn Gui", GenerateBinaryNameFromPath("openvpn-gui.exe"))
-	assert.Equal(t, "Portmaster Core v0-1-2", GenerateBinaryNameFromPath("portmaster-core_v0-1-2.exe"))
-	assert.Equal(t, "Win Store App", GenerateBinaryNameFromPath("WinStore.App.exe"))
-	assert.Equal(t, "Test Script", GenerateBinaryNameFromPath(".test-script"))
-	assert.Equal(t, "Browser Broker", GenerateBinaryNameFromPath("browser_broker.exe"))
-	assert.Equal(t, "Virtual Box VM", GenerateBinaryNameFromPath("VirtualBoxVM"))
-	assert.Equal(t, "Io Elementary Appcenter", GenerateBinaryNameFromPath("io.elementary.appcenter"))
-	assert.Equal(t, "Microsoft Windows Store", GenerateBinaryNameFromPath("Microsoft.WindowsStore"))
-}
-
-func TestCleanFileDescription(t *testing.T) {
-	t.Parallel()
-
-	assert.Equal(t, "Product Name", cleanFileDescription("Product Name. Does this and that."))
-	assert.Equal(t, "Product Name", cleanFileDescription("Product Name - Does this and that."))
-	assert.Equal(t, "Product Name", cleanFileDescription("Product Name / Does this and that."))
-	assert.Equal(t, "Product Name", cleanFileDescription("Product Name :: Does this and that."))
-	assert.Equal(t, "/ Product Name", cleanFileDescription("/ Product Name"))
-	assert.Equal(t, "Product", cleanFileDescription("Product / Name"))
-	assert.Equal(t, "Software 2", cleanFileDescription("Software 2"))
-	assert.Equal(t, "Launcher for Software 2", cleanFileDescription("Launcher for 'Software 2'"))
-	assert.Equal(t, "", cleanFileDescription(". / Name"))
-	assert.Equal(t, "", cleanFileDescription(". "))
-	assert.Equal(t, "", cleanFileDescription("."))
-	assert.Equal(t, "N/A", cleanFileDescription("N/A"))
-
-	assert.Equal(t,
-		"Product Name a Does this and that.",
-		cleanFileDescription("Product Name a Does this and that."),
-	)
-}

utils/osdetail/binmeta_windows.go

@@ -1,78 +0,0 @@
-package osdetail
-
-import (
-	"fmt"
-)
-
-const powershellGetFileDescription = `Get-ItemProperty %q | Select -ExpandProperty VersionInfo | Select -ExpandProperty FileDescription`
-
-// GetBinaryNameFromSystem queries the operating system for a human readable
-// name for the given binary path.
-func GetBinaryNameFromSystem(path string) (string, error) {
-	// Get FileProperties via Powershell call.
-	output, err := RunPowershellCmd(fmt.Sprintf(powershellGetFileDescription, path))
-	if err != nil {
-		return "", fmt.Errorf("failed to get file properties of %s: %s", path, err)
-	}
-
-	// Clean name.
-	binName := cleanFileDescription(string(output))
-	if binName != "" {
-		return binName, nil
-	}
-
-	// Generate a default name as default.
-	return "", ErrNotFound
-}
-
-const powershellGetIcon = `Add-Type -AssemblyName System.Drawing
-$Icon = [System.Drawing.Icon]::ExtractAssociatedIcon(%q)
-$MemoryStream = New-Object System.IO.MemoryStream
-$Icon.save($MemoryStream)
-$Bytes = $MemoryStream.ToArray()
-$MemoryStream.Flush()
-$MemoryStream.Dispose()
-[convert]::ToBase64String($Bytes)`
-
-// TODO: This returns a small and crappy icon.
-
-// Saving a better icon to file works:
-/*
-Add-Type -AssemblyName System.Drawing
-$ImgList = New-Object System.Windows.Forms.ImageList
-$ImgList.ImageSize = New-Object System.Drawing.Size(256,256)
-$ImgList.ColorDepth = 32
-$Icon = [System.Drawing.Icon]::ExtractAssociatedIcon("C:\Program Files (x86)\Mozilla Firefox\firefox.exe")
-$ImgList.Images.Add($Icon);
-$BigIcon = $ImgList.Images.Item(0)
-$BigIcon.Save("test.png")
-*/
-
-// But not saving to a memory stream:
-/*
-Add-Type -AssemblyName System.Drawing
-$ImgList = New-Object System.Windows.Forms.ImageList
-$ImgList.ImageSize = New-Object System.Drawing.Size(256,256)
-$ImgList.ColorDepth = 32
-$Icon = [System.Drawing.Icon]::ExtractAssociatedIcon("C:\Program Files (x86)\Mozilla Firefox\firefox.exe")
-$ImgList.Images.Add($Icon);
-$MemoryStream = New-Object System.IO.MemoryStream
-$BigIcon = $ImgList.Images.Item(0)
-$BigIcon.Save($MemoryStream)
-$Bytes = $MemoryStream.ToArray()
-$MemoryStream.Flush()
-$MemoryStream.Dispose()
-[convert]::ToBase64String($Bytes)
-*/
-
-// GetBinaryIconFromSystem queries the operating system for the associated icon
-// for a given binary path and returns it as a data-URL.
-func GetBinaryIconFromSystem(path string) (string, error) {
-	// Get Associated File Icon via Powershell call.
-	output, err := RunPowershellCmd(fmt.Sprintf(powershellGetIcon, path))
-	if err != nil {
-		return "", fmt.Errorf("failed to get file properties of %s: %s", path, err)
-	}
-
-	return "data:image/png;base64," + string(output), nil
-}

utils/osdetail/test/.gitignore

@@ -1,2 +0,0 @@
-test
-test.exe

utils/osdetail/test/main_windows.go

@@ -1,48 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/safing/portbase/utils/osdetail"
-)
-
-func main() {
-	fmt.Println("Binary Names:")
-	printBinaryName("openvpn-gui.exe", `C:\Program Files\OpenVPN\bin\openvpn-gui.exe`)
-	printBinaryName("firefox.exe", `C:\Program Files (x86)\Mozilla Firefox\firefox.exe`)
-	printBinaryName("powershell.exe", `C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe`)
-	printBinaryName("explorer.exe", `C:\Windows\explorer.exe`)
-	printBinaryName("svchost.exe", `C:\Windows\System32\svchost.exe`)
-
-	fmt.Println("\n\nBinary Icons:")
-	printBinaryIcon("openvpn-gui.exe", `C:\Program Files\OpenVPN\bin\openvpn-gui.exe`)
-	printBinaryIcon("firefox.exe", `C:\Program Files (x86)\Mozilla Firefox\firefox.exe`)
-	printBinaryIcon("powershell.exe", `C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe`)
-	printBinaryIcon("explorer.exe", `C:\Windows\explorer.exe`)
-	printBinaryIcon("svchost.exe", `C:\Windows\System32\svchost.exe`)
-
-	fmt.Println("\n\nSvcHost Service Names:")
-	names, err := osdetail.GetAllServiceNames()
-	if err != nil {
-		panic(err)
-	}
-	fmt.Printf("%+v\n", names)
-}
-
-func printBinaryName(name, path string) {
-	binName, err := osdetail.GetBinaryNameFromSystem(path)
-	if err != nil {
-		fmt.Printf("%s: ERROR: %s\n", name, err)
-	} else {
-		fmt.Printf("%s: %s\n", name, binName)
-	}
-}
-
-func printBinaryIcon(name, path string) {
-	binIcon, err := osdetail.GetBinaryIconFromSystem(path)
-	if err != nil {
-		fmt.Printf("%s: ERROR: %s\n", name, err)
-	} else {
-		fmt.Printf("%s: %s\n", name, binIcon)
-	}
-}

utils/stablepool.go

@@ -2,8 +2,6 @@ package utils
 
 import "sync"
 
-// This file is forked from https://github.com/golang/go/blob/bc593eac2dc63d979a575eccb16c7369a5ff81e0/src/sync/once.go.
-
 // A StablePool is a drop-in replacement for sync.Pool that is slower, but
 // predictable.
 // A StablePool is a set of temporary objects that may be individually saved and