Merge branch 'develop' of github.com:Safing/portbase into develop

api/config.go

@@ -64,7 +64,7 @@ func registerConfig() error {
 	err = config.Register(&config.Option{
 		Name:           "API Keys",
 		Key:            CfgAPIKeys,
-		Description:    "Define API keys for priviledged access to the API. Every entry is a separate API key with respective permissions. Format is `<key>?read=<perm>&write=<perm>`. Permissions are `anyone`, `user` and `admin`, and may be omitted.",
+		Description:    "Define API keys for privileged access to the API. Every entry is a separate API key with respective permissions. Format is `<key>?read=<perm>&write=<perm>`. Permissions are `anyone`, `user` and `admin`, and may be omitted.",
 		Sensitive:      true,
 		OptType:        config.OptTypeStringArray,
 		ExpertiseLevel: config.ExpertiseLevelDeveloper,

api/database.go

@@ -290,7 +290,7 @@ func (api *DatabaseAPI) handleGet(opID []byte, key string) {
 
 	r, err := api.db.Get(key)
 	if err == nil {
-		data, err = marshalRecord(r, true)
+		data, err = MarshalRecord(r, true)
 	}
 	if err != nil {
 		api.send(opID, dbMsgTypeError, err.Error(), nil)
@@ -348,7 +348,7 @@ func (api *DatabaseAPI) processQuery(opID []byte, q *query.Query) (ok bool) {
 			// process query feed
 			if r != nil {
 				// process record
-				data, err := marshalRecord(r, true)
+				data, err := MarshalRecord(r, true)
 				if err != nil {
 					api.send(opID, dbMsgTypeWarning, err.Error(), nil)
 					continue
@@ -425,7 +425,7 @@ func (api *DatabaseAPI) processSub(opID []byte, sub *database.Subscription) {
 			// process sub feed
 			if r != nil {
 				// process record
-				data, err := marshalRecord(r, true)
+				data, err := MarshalRecord(r, true)
 				if err != nil {
 					api.send(opID, dbMsgTypeWarning, err.Error(), nil)
 					continue
@@ -629,9 +629,9 @@ func (api *DatabaseAPI) handleDelete(opID []byte, key string) {
 	api.send(opID, dbMsgTypeSuccess, emptyString, nil)
 }
 
-// marsharlRecords locks and marshals the given record, additionally adding
+// MarshalRecords locks and marshals the given record, additionally adding
 // metadata and returning it as json.
-func marshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) {
+func MarshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) {
 	r.Lock()
 	defer r.Unlock()
 

api/endpoints.go

@@ -452,7 +452,7 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		var rec record.Record
 		rec, err = e.RecordFunc(apiRequest)
 		if err == nil && r != nil {
-			responseData, err = marshalRecord(rec, false)
+			responseData, err = MarshalRecord(rec, false)
 		}
 
 	case e.HandlerFunc != nil:

api/main.go

@@ -1,7 +1,6 @@
 package api
 
 import (
-	"context"
 	"encoding/json"
 	"errors"
 	"flag"
@@ -58,7 +57,7 @@ func prep() error {
 }
 
 func start() error {
-	go Serve()
+	startServer()
 
 	_ = updateAPIKeys(module.Ctx, nil)
 	err := module.RegisterEventHook("config", "config change", "update API keys", updateAPIKeys)
@@ -75,10 +74,7 @@ func start() error {
 }
 
 func stop() error {
-	if server != nil {
+	return stopServer()
-		return server.Shutdown(context.Background())
-	}
-	return nil
 }
 
 func exportEndpointsCmd() error {

api/router.go

@@ -18,6 +18,9 @@ import (
 	"github.com/safing/portbase/utils"
 )
 
+// EnableServer defines if the HTTP server should be started.
+const EnableServer = true
+
 var (
 	// mainMux is the main mux router.
 	mainMux = mux.NewRouter()
@@ -48,15 +51,38 @@ func RegisterHandleFunc(path string, handleFunc func(http.ResponseWriter, *http.
 	return mainMux.HandleFunc(path, handleFunc)
 }
 
-// Serve starts serving the API endpoint.
+func startServer() {
-func Serve() {
+	// Check if server is enabled.
-	// configure server
+	if !EnableServer {
+		return
+	}
+
+	// Configure server.
 	server.Addr = listenAddressConfig()
 	server.Handler = &mainHandler{
 		// TODO: mainMux should not be modified anymore.
 		mux: mainMux,
 	}
 
+	// Start server manager.
+	module.StartServiceWorker("http server manager", 0, serverManager)
+}
+
+func stopServer() error {
+	// Check if server is enabled.
+	if !EnableServer {
+		return nil
+	}
+
+	if server.Addr != "" {
+		return server.Shutdown(context.Background())
+	}
+
+	return nil
+}
+
+// Serve starts serving the API endpoint.
+func serverManager(_ context.Context) error {
 	// start serving
 	log.Infof("api: starting to listen on %s", server.Addr)
 	backoffDuration := 10 * time.Second
@@ -67,7 +93,7 @@ func Serve() {
 		})
 		// return on shutdown error
 		if errors.Is(err, http.ErrServerClosed) {
-			return
+			return nil
 		}
 		// log error and restart
 		log.Errorf("api: http endpoint failed: %s - restarting in %s", err, backoffDuration)

go.mod

@@ -1,6 +1,6 @@
 module github.com/safing/portbase
 
-go 1.15
+go 1.19
 
 require (
 	github.com/VictoriaMetrics/metrics v1.23.1

log/output.go

@@ -17,10 +17,10 @@ type (
 
 	// AdapterFunc is a convenience type for implementing
 	// Adapter.
-	AdapterFunc func(msg Message, duplciates uint64)
+	AdapterFunc func(msg Message, duplicates uint64)
 
 	// FormatFunc formats msg into a string.
-	FormatFunc func(msg Message, duplciates uint64) string
+	FormatFunc func(msg Message, duplicates uint64) string
 
 	// SimpleFileAdapter implements Adapter and writes all
 	// messages to File.

modules/subsystems/module.go

@@ -64,7 +64,7 @@ func prep() error {
 	}
 
 	// We need to listen for configuration changes so we can
-	// start/stop dependend modules in case a subsystem is
+	// start/stop depended modules in case a subsystem is
 	// (de-)activated.
 	if err := module.RegisterEventHook(
 		"config",

modules/worker.go

@@ -125,6 +125,8 @@ func (m *Module) runWorker(name string, fn func(context.Context) error) (err err
 	}()
 
 	// run
+	// TODO: get cancel func for worker context and cancel when worker is done.
+	// This ensure that when the worker passes its context to another (async) function, it will also be shutdown when the worker finished or dies.
 	err = fn(m.Ctx)
 	return
 }

updater/fetch.go

@@ -145,7 +145,7 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client,
 		}
 	}
 
-	log.Infof("%s: fetched %s (stored to %s)", reg.Name, downloadURL, rv.storagePath())
+	log.Debugf("%s: fetched %s and stored to %s", reg.Name, downloadURL, rv.storagePath())
 	return nil
 }
 
@@ -223,7 +223,7 @@ func (reg *ResourceRegistry) fetchMissingSig(ctx context.Context, client *http.C
 		}
 	}
 
-	log.Infof("%s: fetched %s (stored to %s)", reg.Name, rv.versionedSigPath(), rv.storageSigPath())
+	log.Debugf("%s: fetched %s and stored to %s", reg.Name, rv.versionedSigPath(), rv.storageSigPath())
 	return nil
 }
 

updater/get.go

@@ -52,6 +52,10 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
 		return nil, fmt.Errorf("could not prepare tmp directory for download: %w", err)
 	}
 
+	// Start registry operation.
+	reg.state.StartOperation(StateFetching)
+	defer reg.state.EndOperation()
+
 	// download file
 	log.Tracef("%s: starting download of %s", reg.Name, file.versionedPath)
 	client := &http.Client{}
@@ -69,3 +73,19 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
 	log.Warningf("%s: failed to download %s: %s", reg.Name, file.versionedPath, err)
 	return nil, err
 }
+
+// GetVersion returns the selected version of the given identifier.
+// The returned resource version may not be modified.
+func (reg *ResourceRegistry) GetVersion(identifier string) (*ResourceVersion, error) {
+	reg.RLock()
+	res, ok := reg.resources[identifier]
+	reg.RUnlock()
+	if !ok {
+		return nil, ErrNotFound
+	}
+
+	res.Lock()
+	defer res.Unlock()
+
+	return res.SelectedVersion, nil
+}

updater/indexes.go

@@ -27,6 +27,9 @@ type Index struct {
 	// not.
 	PreRelease bool
 
+	// AutoDownload specifies whether new versions should be automatically downloaded.
+	AutoDownload bool
+
 	// LastRelease holds the time of the last seen release of this index.
 	LastRelease time.Time
 }

updater/registry.go

@@ -25,6 +25,7 @@ type ResourceRegistry struct {
 	storageDir *utils.DirStructure
 	tmpDir     *utils.DirStructure
 	indexes    []*Index
+	state      *RegistryState
 
 	resources        map[string]*Resource
 	UpdateURLs       []string
@@ -44,6 +45,11 @@ type ResourceRegistry struct {
 	UsePreReleases bool
 	DevMode        bool
 	Online         bool
+
+	// StateNotifyFunc may be set to receive any changes to the registry state.
+	// The specified function may lock the state, but may not block or take a
+	// lot of time.
+	StateNotifyFunc func(*RegistryState)
 }
 
 // AddIndex adds a new index to the resource registry.
@@ -61,6 +67,18 @@ func (reg *ResourceRegistry) AddIndex(idx Index) {
 	reg.indexes = append(reg.indexes, &idx)
 }
 
+// PreInitUpdateState sets the initial update state of the registry before initialization.
+func (reg *ResourceRegistry) PreInitUpdateState(s UpdateState) error {
+	if reg.state != nil {
+		return errors.New("registry already initialized")
+	}
+
+	reg.state = &RegistryState{
+		Updates: s,
+	}
+	return nil
+}
+
 // Initialize initializes a raw registry struct and makes it ready for usage.
 func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
 	// check if storage dir is available
@@ -78,6 +96,11 @@ func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
 	reg.storageDir = storageDir
 	reg.tmpDir = storageDir.ChildDir("tmp", 0o0700)
 	reg.resources = make(map[string]*Resource)
+	if reg.state == nil {
+		reg.state = &RegistryState{}
+	}
+	reg.state.ID = StateReady
+	reg.state.reg = reg
 
 	// remove tmp dir to delete old entries
 	err = reg.Cleanup()
@@ -147,32 +170,34 @@ func (reg *ResourceRegistry) SetUsePreReleases(yes bool) {
 }
 
 // AddResource adds a resource to the registry. Does _not_ select new version.
-func (reg *ResourceRegistry) AddResource(identifier, version string, available, currentRelease, preRelease bool) error {
+func (reg *ResourceRegistry) AddResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error {
 	reg.Lock()
 	defer reg.Unlock()
 
-	err := reg.addResource(identifier, version, available, currentRelease, preRelease)
+	err := reg.addResource(identifier, version, index, available, currentRelease, preRelease)
 	return err
 }
 
-func (reg *ResourceRegistry) addResource(identifier, version string, available, currentRelease, preRelease bool) error {
+func (reg *ResourceRegistry) addResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error {
 	res, ok := reg.resources[identifier]
 	if !ok {
 		res = reg.newResource(identifier)
 		reg.resources[identifier] = res
 	}
+	res.Index = index
+
 	return res.AddVersion(version, available, currentRelease, preRelease)
 }
 
 // AddResources adds resources to the registry. Errors are logged, the last one is returned. Despite errors, non-failing resources are still added. Does _not_ select new versions.
-func (reg *ResourceRegistry) AddResources(versions map[string]string, available, currentRelease, preRelease bool) error {
+func (reg *ResourceRegistry) AddResources(versions map[string]string, index *Index, available, currentRelease, preRelease bool) error {
 	reg.Lock()
 	defer reg.Unlock()
 
 	// add versions and their flags to registry
 	var lastError error
 	for identifier, version := range versions {
-		lastError = reg.addResource(identifier, version, available, currentRelease, preRelease)
+		lastError = reg.addResource(identifier, version, index, available, currentRelease, preRelease)
 		if lastError != nil {
 			log.Warningf("%s: failed to add resource %s: %s", reg.Name, identifier, lastError)
 		}

updater/resource.go

@@ -55,6 +55,10 @@ type Resource struct {
 
 	// VerificationOptions holds the verification options for this resource.
 	VerificationOptions *VerificationOptions
+
+	// Index holds a reference to the index this resource was last defined in.
+	// Will be nil if resource was only found on disk.
+	Index *Index
 }
 
 // ResourceVersion represents a single version of a resource.
@@ -89,7 +93,7 @@ func (rv *ResourceVersion) String() string {
 	return rv.VersionNumber
 }
 
-// SemVer returns the semantiv version of the resource.
+// SemVer returns the semantic version of the resource.
 func (rv *ResourceVersion) SemVer() *semver.Version {
 	return rv.semVer
 }

updater/state.go

@@ -0,0 +1,180 @@
+package updater
+
+import (
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/safing/portbase/utils"
+)
+
+// Registry States.
+const (
+	StateReady       = "ready"       // Default idle state.
+	StateChecking    = "checking"    // Downloading indexes.
+	StateDownloading = "downloading" // Downloading updates.
+	StateFetching    = "fetching"    // Fetching a single file.
+)
+
+// RegistryState describes the registry state.
+type RegistryState struct {
+	sync.Mutex
+	reg *ResourceRegistry
+
+	// ID holds the ID of the state the registry is currently in.
+	ID string
+
+	// Details holds further information about the current state.
+	Details any
+
+	// Updates holds generic information about the current status of pending
+	// and recently downloaded updates.
+	Updates UpdateState
+
+	// operationLock locks the operation of any state changing operation.
+	// This is separate from the registry lock, which locks access to the
+	// registry struct.
+	operationLock sync.Mutex
+}
+
+// StateDownloadingDetails holds details of the downloading state.
+type StateDownloadingDetails struct {
+	// Resources holds the resource IDs that are being downloaded.
+	Resources []string
+
+	// FinishedUpTo holds the index of Resources that is currently being
+	// downloaded. Previous resources have finished downloading.
+	FinishedUpTo int
+}
+
+// UpdateState holds generic information about the current status of pending
+// and recently downloaded updates.
+type UpdateState struct {
+	// LastCheckAt holds the time of the last update check.
+	LastCheckAt *time.Time
+	// LastCheckError holds the error of the last check.
+	LastCheckError error
+	// PendingDownload holds the resources that are pending download.
+	PendingDownload []string
+
+	// LastDownloadAt holds the time when resources were downloaded the last time.
+	LastDownloadAt *time.Time
+	// LastDownloadError holds the error of the last download.
+	LastDownloadError error
+	// LastDownload holds the resources that we downloaded the last time updates
+	// were downloaded.
+	LastDownload []string
+
+	// LastSuccessAt holds the time of the last successful update (check).
+	LastSuccessAt *time.Time
+}
+
+// GetState returns the current registry state.
+// The returned data must not be modified.
+func (reg *ResourceRegistry) GetState() RegistryState {
+	reg.state.Lock()
+	defer reg.state.Unlock()
+
+	return RegistryState{
+		ID:      reg.state.ID,
+		Details: reg.state.Details,
+		Updates: reg.state.Updates,
+	}
+}
+
+// StartOperation starts an operation.
+func (s *RegistryState) StartOperation(id string) bool {
+	defer s.notify()
+
+	s.operationLock.Lock()
+
+	s.Lock()
+	defer s.Unlock()
+
+	s.ID = id
+	return true
+}
+
+// UpdateOperationDetails updates the details of an operation.
+// The supplied struct should be a copy and must not be changed after calling
+// this function.
+func (s *RegistryState) UpdateOperationDetails(details any) {
+	defer s.notify()
+
+	s.Lock()
+	defer s.Unlock()
+
+	s.Details = details
+}
+
+// EndOperation ends an operation.
+func (s *RegistryState) EndOperation() {
+	defer s.notify()
+	defer s.operationLock.Unlock()
+
+	s.Lock()
+	defer s.Unlock()
+
+	s.ID = StateReady
+	s.Details = nil
+}
+
+// ReportUpdateCheck reports an update check to the registry state.
+func (s *RegistryState) ReportUpdateCheck(pendingDownload []string, failed error) {
+	defer s.notify()
+
+	sort.Strings(pendingDownload)
+
+	s.Lock()
+	defer s.Unlock()
+
+	now := time.Now()
+	s.Updates.LastCheckAt = &now
+	s.Updates.LastCheckError = failed
+	s.Updates.PendingDownload = pendingDownload
+
+	if failed == nil {
+		s.Updates.LastSuccessAt = &now
+	}
+}
+
+// ReportDownloads reports downloaded updates to the registry state.
+func (s *RegistryState) ReportDownloads(downloaded []string, failed error) {
+	defer s.notify()
+
+	sort.Strings(downloaded)
+
+	s.Lock()
+	defer s.Unlock()
+
+	now := time.Now()
+	s.Updates.LastDownloadAt = &now
+	s.Updates.LastDownloadError = failed
+	s.Updates.LastDownload = downloaded
+
+	// Remove downloaded resources from the pending list.
+	if len(s.Updates.PendingDownload) > 0 {
+		newPendingDownload := make([]string, 0, len(s.Updates.PendingDownload))
+		for _, pending := range s.Updates.PendingDownload {
+			if !utils.StringInSlice(downloaded, pending) {
+				newPendingDownload = append(newPendingDownload, pending)
+			}
+		}
+		s.Updates.PendingDownload = newPendingDownload
+	}
+
+	if failed == nil {
+		s.Updates.LastSuccessAt = &now
+	}
+}
+
+func (s *RegistryState) notify() {
+	switch {
+	case s.reg == nil:
+		return
+	case s.reg.StateNotifyFunc == nil:
+		return
+	}
+
+	s.reg.StateNotifyFunc(s)
+}

updater/storage.go

@@ -79,7 +79,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error {
 		}
 
 		// save
-		err = reg.AddResource(identifier, version, true, false, false)
+		err = reg.AddResource(identifier, version, nil, true, false, false)
 		if err != nil {
 			lastError = fmt.Errorf("%s: could not get add resource %s v%s: %w", reg.Name, identifier, version, err)
 			log.Warning(lastError.Error())
@@ -178,7 +178,7 @@ func (reg *ResourceRegistry) loadIndexFile(idx *Index) error {
 	}
 
 	// Add index releases to available resources.
-	err = reg.AddResources(indexFile.Releases, false, true, idx.PreRelease)
+	err = reg.AddResources(indexFile.Releases, idx, false, true, idx.PreRelease)
 	if err != nil {
 		log.Warningf("%s: failed to add resource: %s", reg.Name, err)
 	}

updater/updating.go

@@ -8,7 +8,8 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
+
+	"golang.org/x/exp/slices"
 
 	"github.com/safing/jess/filesig"
 	"github.com/safing/jess/lhash"
@@ -22,6 +23,10 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
 	var lastErr error
 	var anySuccess bool
 
+	// Start registry operation.
+	reg.state.StartOperation(StateChecking)
+	defer reg.state.EndOperation()
+
 	client := &http.Client{}
 	for _, idx := range reg.getIndexes() {
 		if err := reg.downloadIndex(ctx, client, idx); err != nil {
@@ -32,9 +37,20 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
 		}
 	}
 
+	// If all indexes failed to update, fail.
 	if !anySuccess {
-		return fmt.Errorf("failed to update all indexes, last error was: %w", lastErr)
+		err := fmt.Errorf("failed to update all indexes, last error was: %w", lastErr)
+		reg.state.ReportUpdateCheck(nil, err)
+		return err
 	}
+
+	// Get pending resources and update status.
+	pendingResourceVersions, _ := reg.GetPendingDownloads(true, false)
+	reg.state.ReportUpdateCheck(
+		identifiersFromResourceVersions(pendingResourceVersions),
+		nil,
+	)
+
 	return nil
 }
 
@@ -127,7 +143,7 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
 		}
 
 		// add resources to registry
-		err = reg.AddResources(cleanedData, false, true, idx.PreRelease)
+		err = reg.AddResources(cleanedData, idx, false, true, idx.PreRelease)
 		if err != nil {
 			log.Warningf("%s: failed to add resources: %s", reg.Name, err)
 		}
@@ -167,37 +183,17 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
 }
 
 // DownloadUpdates checks if updates are available and downloads updates of used components.
-func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
+func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context, automaticOnly bool) error {
-	// create list of downloads
+	// Start registry operation.
-	var toUpdate []*ResourceVersion
+	reg.state.StartOperation(StateDownloading)
-	var missingSigs []*ResourceVersion
+	defer reg.state.EndOperation()
-	reg.RLock()
-	for _, res := range reg.resources {
-		res.Lock()
 
-		// check if we want to download
+	// Get pending updates.
-		if res.inUse() ||
+	toUpdate, missingSigs := reg.GetPendingDownloads(!automaticOnly, true)
-			res.available() || // resource was used in the past
+	downloadDetailsResources := identifiersFromResourceVersions(toUpdate)
-			utils.StringInSlice(reg.MandatoryUpdates, res.Identifier) { // resource is mandatory
+	reg.state.UpdateOperationDetails(&StateDownloadingDetails{
-
+		Resources: downloadDetailsResources,
-			// add all non-available and eligible versions to update queue
+	})
-			for _, rv := range res.Versions {
-				switch {
-				case !rv.CurrentRelease:
-					// We are not interested in older releases.
-				case !rv.Available:
-					// File is not available.
-					toUpdate = append(toUpdate, rv)
-				case !rv.SigAvailable && res.VerificationOptions != nil:
-					// File signature is not available and verification is enabled.
-					missingSigs = append(missingSigs, rv)
-				}
-			}
-		}
-
-		res.Unlock()
-	}
-	reg.RUnlock()
 
 	// nothing to update
 	if len(toUpdate) == 0 && len(missingSigs) == 0 {
@@ -211,63 +207,153 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
 	}
 
 	// download updates
-	log.Infof("%s: starting to download %d updates in parallel", reg.Name, len(toUpdate)+len(missingSigs))
+	log.Infof("%s: starting to download %d updates", reg.Name, len(toUpdate))
-	var wg sync.WaitGroup
-
-	wg.Add(len(toUpdate) + len(missingSigs))
 	client := &http.Client{}
+	var reportError error
 
-	for idx := range toUpdate {
+	for i, rv := range toUpdate {
-		go func(rv *ResourceVersion) {
+		log.Infof(
+			"%s: downloading update [%d/%d]: %s version %s",
+			reg.Name,
+			i+1, len(toUpdate),
+			rv.resource.Identifier, rv.VersionNumber,
+		)
 		var err error
-
-			defer wg.Done()
-			defer func() {
-				if x := recover(); x != nil {
-					log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x)
-				}
-			}()
-
 		for tries := 0; tries < 3; tries++ {
 			err = reg.fetchFile(ctx, client, rv, tries)
 			if err == nil {
+				// Update resource version state.
+				rv.resource.Lock()
 				rv.Available = true
-					return
+				if rv.resource.VerificationOptions != nil {
+					rv.SigAvailable = true
+				}
+				rv.resource.Unlock()
+
+				break
 			}
 		}
 		if err != nil {
-				log.Warningf("%s: failed to download %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
+			reportError := fmt.Errorf("failed to download %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
-			}
+			log.Warningf("%s: %s", reg.Name, reportError)
-		}(toUpdate[idx])
 		}
 
-	for idx := range missingSigs {
+		reg.state.UpdateOperationDetails(&StateDownloadingDetails{
-		go func(rv *ResourceVersion) {
+			Resources:    downloadDetailsResources,
+			FinishedUpTo: i + 1,
+		})
+	}
+
+	if len(missingSigs) > 0 {
+		log.Infof("%s: downloading %d missing signatures", reg.Name, len(missingSigs))
+
+		for _, rv := range missingSigs {
 			var err error
-
-			defer wg.Done()
-			defer func() {
-				if x := recover(); x != nil {
-					log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x)
-				}
-			}()
-
 			for tries := 0; tries < 3; tries++ {
 				err = reg.fetchMissingSig(ctx, client, rv, tries)
 				if err == nil {
+					// Update resource version state.
+					rv.resource.Lock()
 					rv.SigAvailable = true
-					return
+					rv.resource.Unlock()
+
+					break
 				}
 			}
 			if err != nil {
-				log.Warningf("%s: failed to download missing sig of %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
+				reportError := fmt.Errorf("failed to download missing sig of %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
+				log.Warningf("%s: %s", reg.Name, reportError)
+			}
 		}
-		}(missingSigs[idx])
 	}
 
-	wg.Wait()
+	reg.state.ReportDownloads(
-
+		downloadDetailsResources,
+		reportError,
+	)
 	log.Infof("%s: finished downloading updates", reg.Name)
 
 	return nil
 }
+
+// DownloadUpdates checks if updates are available and downloads updates of used components.
+
+// GetPendingDownloads returns the list of pending downloads.
+// If manual is set, indexes with AutoDownload=false will be checked.
+// If auto is set, indexes with AutoDownload=true will be checked.
+func (reg *ResourceRegistry) GetPendingDownloads(manual, auto bool) (resources, sigs []*ResourceVersion) {
+	reg.RLock()
+	defer reg.RUnlock()
+
+	// create list of downloads
+	var toUpdate []*ResourceVersion
+	var missingSigs []*ResourceVersion
+
+	for _, res := range reg.resources {
+		func() {
+			res.Lock()
+			defer res.Unlock()
+
+			// Skip resources without index or indexes that should not be reported
+			// according to parameters.
+			switch {
+			case res.Index == nil:
+				// Cannot download if resource is not part of an index.
+				return
+			case manual && !res.Index.AutoDownload:
+				// Manual update report and index is not auto-download.
+			case auto && res.Index.AutoDownload:
+				// Auto update report and index is auto-download.
+			default:
+				// Resource should not be reported.
+				return
+			}
+
+			// Skip resources we don't need.
+			switch {
+			case res.inUse():
+				// Update if resource is in use.
+			case res.available():
+				// Update if resource is available locally, ie. was used in the past.
+			case utils.StringInSlice(reg.MandatoryUpdates, res.Identifier):
+				// Update is set as mandatory.
+			default:
+				// Resource does not need to be updated.
+				return
+			}
+
+			// Go through all versions until we find versions that need updating.
+			for _, rv := range res.Versions {
+				switch {
+				case !rv.CurrentRelease:
+					// We are not interested in older releases.
+				case !rv.Available:
+					// File not available locally, download!
+					toUpdate = append(toUpdate, rv)
+				case !rv.SigAvailable && res.VerificationOptions != nil:
+					// File signature is not available and verification is enabled, download signature!
+					missingSigs = append(missingSigs, rv)
+				}
+			}
+		}()
+	}
+
+	slices.SortFunc[*ResourceVersion](toUpdate, func(a, b *ResourceVersion) bool {
+		return a.resource.Identifier < b.resource.Identifier
+	})
+	slices.SortFunc[*ResourceVersion](missingSigs, func(a, b *ResourceVersion) bool {
+		return a.resource.Identifier < b.resource.Identifier
+	})
+
+	return toUpdate, missingSigs
+}
+
+func identifiersFromResourceVersions(resourceVersions []*ResourceVersion) []string {
+	identifiers := make([]string, len(resourceVersions))
+
+	for i, rv := range resourceVersions {
+		identifiers[i] = rv.resource.Identifier
+	}
+
+	return identifiers
+}

utils/debug/debug.go

@@ -2,14 +2,11 @@ package debug
 
 import (
 	"bytes"
-	"context"
 	"fmt"
 	"runtime/pprof"
 	"strings"
 	"time"
 
-	"github.com/shirou/gopsutil/host"
-
 	"github.com/safing/portbase/info"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
@@ -94,39 +91,6 @@ func (di *Info) AddVersionInfo() {
 	)
 }
 
-// AddPlatformInfo adds OS and platform information.
-func (di *Info) AddPlatformInfo(ctx context.Context) {
-	// Get information from the system.
-	info, err := host.InfoWithContext(ctx)
-	if err != nil {
-		di.AddSection(
-			"Platform Information",
-			NoFlags,
-			fmt.Sprintf("Failed to get: %s", err),
-		)
-		return
-	}
-
-	// Check if we want to add virtulization information.
-	var virtInfo string
-	if info.VirtualizationRole == "guest" {
-		if info.VirtualizationSystem != "" {
-			virtInfo = fmt.Sprintf("VM: %s", info.VirtualizationSystem)
-		} else {
-			virtInfo = "VM: unidentified"
-		}
-	}
-
-	// Add section.
-	di.AddSection(
-		fmt.Sprintf("Platform: %s %s", info.Platform, info.PlatformVersion),
-		UseCodeSection|AddContentLineBreaks,
-		fmt.Sprintf("System: %s %s (%s) %s", info.Platform, info.OS, info.PlatformFamily, info.PlatformVersion),
-		fmt.Sprintf("Kernel: %s %s", info.KernelVersion, info.KernelArch),
-		virtInfo,
-	)
-}
-
 // AddGoroutineStack adds the current goroutine stack.
 func (di *Info) AddGoroutineStack() {
 	buf := new(bytes.Buffer)

utils/debug/debug_android.go

@@ -0,0 +1,31 @@
+package debug
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/safing/portmaster-android/go/app_interface"
+)
+
+// AddPlatformInfo adds OS and platform information.
+func (di *Info) AddPlatformInfo(_ context.Context) {
+	// Get information from the system.
+	info, err := app_interface.GetPlatformInfo()
+	if err != nil {
+		di.AddSection(
+			"Platform Information",
+			NoFlags,
+			fmt.Sprintf("Failed to get: %s", err),
+		)
+		return
+	}
+
+	// Add section.
+	di.AddSection(
+		fmt.Sprintf("Platform: Android"),
+		UseCodeSection|AddContentLineBreaks,
+		fmt.Sprintf("SDK: %d", info.SDK),
+		fmt.Sprintf("Device: %s %s (%s)", info.Manufacturer, info.Brand, info.Board),
+		fmt.Sprintf("App: %s: %s(%d) %s", info.ApplicationID, info.VersionName, info.VersionCode, info.BuildType))
+
+}

utils/debug/debug_default.go

@@ -0,0 +1,43 @@
+//go:build !android
+
+package debug
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/shirou/gopsutil/host"
+)
+
+// AddPlatformInfo adds OS and platform information.
+func (di *Info) AddPlatformInfo(ctx context.Context) {
+	// Get information from the system.
+	info, err := host.InfoWithContext(ctx)
+	if err != nil {
+		di.AddSection(
+			"Platform Information",
+			NoFlags,
+			fmt.Sprintf("Failed to get: %s", err),
+		)
+		return
+	}
+
+	// Check if we want to add virtulization information.
+	var virtInfo string
+	if info.VirtualizationRole == "guest" {
+		if info.VirtualizationSystem != "" {
+			virtInfo = fmt.Sprintf("VM: %s", info.VirtualizationSystem)
+		} else {
+			virtInfo = "VM: unidentified"
+		}
+	}
+
+	// Add section.
+	di.AddSection(
+		fmt.Sprintf("Platform: %s %s", info.Platform, info.PlatformVersion),
+		UseCodeSection|AddContentLineBreaks,
+		fmt.Sprintf("System: %s %s (%s) %s", info.Platform, info.OS, info.PlatformFamily, info.PlatformVersion),
+		fmt.Sprintf("Kernel: %s %s", info.KernelVersion, info.KernelArch),
+		virtInfo,
+	)
+}