diff --git a/api/config.go b/api/config.go index 7b037d8..9e9fac1 100644 --- a/api/config.go +++ b/api/config.go @@ -64,7 +64,7 @@ func registerConfig() error { err = config.Register(&config.Option{ Name: "API Keys", Key: CfgAPIKeys, - Description: "Define API keys for priviledged access to the API. Every entry is a separate API key with respective permissions. Format is `?read=&write=`. Permissions are `anyone`, `user` and `admin`, and may be omitted.", + Description: "Define API keys for privileged access to the API. Every entry is a separate API key with respective permissions. Format is `?read=&write=`. Permissions are `anyone`, `user` and `admin`, and may be omitted.", Sensitive: true, OptType: config.OptTypeStringArray, ExpertiseLevel: config.ExpertiseLevelDeveloper, diff --git a/api/database.go b/api/database.go index a6d7809..804dc1a 100644 --- a/api/database.go +++ b/api/database.go @@ -290,7 +290,7 @@ func (api *DatabaseAPI) handleGet(opID []byte, key string) { r, err := api.db.Get(key) if err == nil { - data, err = marshalRecord(r, true) + data, err = MarshalRecord(r, true) } if err != nil { api.send(opID, dbMsgTypeError, err.Error(), nil) @@ -348,7 +348,7 @@ func (api *DatabaseAPI) processQuery(opID []byte, q *query.Query) (ok bool) { // process query feed if r != nil { // process record - data, err := marshalRecord(r, true) + data, err := MarshalRecord(r, true) if err != nil { api.send(opID, dbMsgTypeWarning, err.Error(), nil) continue @@ -425,7 +425,7 @@ func (api *DatabaseAPI) processSub(opID []byte, sub *database.Subscription) { // process sub feed if r != nil { // process record - data, err := marshalRecord(r, true) + data, err := MarshalRecord(r, true) if err != nil { api.send(opID, dbMsgTypeWarning, err.Error(), nil) continue @@ -629,9 +629,9 @@ func (api *DatabaseAPI) handleDelete(opID []byte, key string) { api.send(opID, dbMsgTypeSuccess, emptyString, nil) } -// marsharlRecords locks and marshals the given record, additionally adding +// MarshalRecords locks and marshals the given record, additionally adding // metadata and returning it as json. -func marshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) { +func MarshalRecord(r record.Record, withDSDIdentifier bool) ([]byte, error) { r.Lock() defer r.Unlock() diff --git a/api/endpoints.go b/api/endpoints.go index 9e2586a..3a005af 100644 --- a/api/endpoints.go +++ b/api/endpoints.go @@ -452,7 +452,7 @@ func (e *Endpoint) ServeHTTP(w http.ResponseWriter, r *http.Request) { var rec record.Record rec, err = e.RecordFunc(apiRequest) if err == nil && r != nil { - responseData, err = marshalRecord(rec, false) + responseData, err = MarshalRecord(rec, false) } case e.HandlerFunc != nil: diff --git a/api/main.go b/api/main.go index 68172c8..b51e164 100644 --- a/api/main.go +++ b/api/main.go @@ -1,7 +1,6 @@ package api import ( - "context" "encoding/json" "errors" "flag" @@ -58,7 +57,7 @@ func prep() error { } func start() error { - go Serve() + startServer() _ = updateAPIKeys(module.Ctx, nil) err := module.RegisterEventHook("config", "config change", "update API keys", updateAPIKeys) @@ -75,10 +74,7 @@ func start() error { } func stop() error { - if server != nil { - return server.Shutdown(context.Background()) - } - return nil + return stopServer() } func exportEndpointsCmd() error { diff --git a/api/router.go b/api/router.go index c12d20b..3cb4cc6 100644 --- a/api/router.go +++ b/api/router.go @@ -18,6 +18,9 @@ import ( "github.com/safing/portbase/utils" ) +// EnableServer defines if the HTTP server should be started. +const EnableServer = true + var ( // mainMux is the main mux router. mainMux = mux.NewRouter() @@ -48,15 +51,38 @@ func RegisterHandleFunc(path string, handleFunc func(http.ResponseWriter, *http. return mainMux.HandleFunc(path, handleFunc) } -// Serve starts serving the API endpoint. -func Serve() { - // configure server +func startServer() { + // Check if server is enabled. + if !EnableServer { + return + } + + // Configure server. server.Addr = listenAddressConfig() server.Handler = &mainHandler{ // TODO: mainMux should not be modified anymore. mux: mainMux, } + // Start server manager. + module.StartServiceWorker("http server manager", 0, serverManager) +} + +func stopServer() error { + // Check if server is enabled. + if !EnableServer { + return nil + } + + if server.Addr != "" { + return server.Shutdown(context.Background()) + } + + return nil +} + +// Serve starts serving the API endpoint. +func serverManager(_ context.Context) error { // start serving log.Infof("api: starting to listen on %s", server.Addr) backoffDuration := 10 * time.Second @@ -67,7 +93,7 @@ func Serve() { }) // return on shutdown error if errors.Is(err, http.ErrServerClosed) { - return + return nil } // log error and restart log.Errorf("api: http endpoint failed: %s - restarting in %s", err, backoffDuration) diff --git a/go.mod b/go.mod index b6919b1..46deb43 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/safing/portbase -go 1.15 +go 1.19 require ( github.com/VictoriaMetrics/metrics v1.23.1 diff --git a/log/output.go b/log/output.go index f3831c2..d8a29a4 100644 --- a/log/output.go +++ b/log/output.go @@ -17,10 +17,10 @@ type ( // AdapterFunc is a convenience type for implementing // Adapter. - AdapterFunc func(msg Message, duplciates uint64) + AdapterFunc func(msg Message, duplicates uint64) // FormatFunc formats msg into a string. - FormatFunc func(msg Message, duplciates uint64) string + FormatFunc func(msg Message, duplicates uint64) string // SimpleFileAdapter implements Adapter and writes all // messages to File. diff --git a/modules/subsystems/module.go b/modules/subsystems/module.go index 31d129b..79ea45a 100644 --- a/modules/subsystems/module.go +++ b/modules/subsystems/module.go @@ -64,7 +64,7 @@ func prep() error { } // We need to listen for configuration changes so we can - // start/stop dependend modules in case a subsystem is + // start/stop depended modules in case a subsystem is // (de-)activated. if err := module.RegisterEventHook( "config", diff --git a/modules/worker.go b/modules/worker.go index 5148d1d..dcf1697 100644 --- a/modules/worker.go +++ b/modules/worker.go @@ -125,6 +125,8 @@ func (m *Module) runWorker(name string, fn func(context.Context) error) (err err }() // run + // TODO: get cancel func for worker context and cancel when worker is done. + // This ensure that when the worker passes its context to another (async) function, it will also be shutdown when the worker finished or dies. err = fn(m.Ctx) return } diff --git a/updater/fetch.go b/updater/fetch.go index 753d909..6b306c0 100644 --- a/updater/fetch.go +++ b/updater/fetch.go @@ -145,7 +145,7 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client, } } - log.Infof("%s: fetched %s (stored to %s)", reg.Name, downloadURL, rv.storagePath()) + log.Debugf("%s: fetched %s and stored to %s", reg.Name, downloadURL, rv.storagePath()) return nil } @@ -223,7 +223,7 @@ func (reg *ResourceRegistry) fetchMissingSig(ctx context.Context, client *http.C } } - log.Infof("%s: fetched %s (stored to %s)", reg.Name, rv.versionedSigPath(), rv.storageSigPath()) + log.Debugf("%s: fetched %s and stored to %s", reg.Name, rv.versionedSigPath(), rv.storageSigPath()) return nil } diff --git a/updater/get.go b/updater/get.go index e235e00..a311682 100644 --- a/updater/get.go +++ b/updater/get.go @@ -52,6 +52,10 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) { return nil, fmt.Errorf("could not prepare tmp directory for download: %w", err) } + // Start registry operation. + reg.state.StartOperation(StateFetching) + defer reg.state.EndOperation() + // download file log.Tracef("%s: starting download of %s", reg.Name, file.versionedPath) client := &http.Client{} @@ -69,3 +73,19 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) { log.Warningf("%s: failed to download %s: %s", reg.Name, file.versionedPath, err) return nil, err } + +// GetVersion returns the selected version of the given identifier. +// The returned resource version may not be modified. +func (reg *ResourceRegistry) GetVersion(identifier string) (*ResourceVersion, error) { + reg.RLock() + res, ok := reg.resources[identifier] + reg.RUnlock() + if !ok { + return nil, ErrNotFound + } + + res.Lock() + defer res.Unlock() + + return res.SelectedVersion, nil +} diff --git a/updater/indexes.go b/updater/indexes.go index 35491b2..81a373a 100644 --- a/updater/indexes.go +++ b/updater/indexes.go @@ -27,6 +27,9 @@ type Index struct { // not. PreRelease bool + // AutoDownload specifies whether new versions should be automatically downloaded. + AutoDownload bool + // LastRelease holds the time of the last seen release of this index. LastRelease time.Time } diff --git a/updater/registry.go b/updater/registry.go index 6c0fbd0..b05d1ad 100644 --- a/updater/registry.go +++ b/updater/registry.go @@ -25,6 +25,7 @@ type ResourceRegistry struct { storageDir *utils.DirStructure tmpDir *utils.DirStructure indexes []*Index + state *RegistryState resources map[string]*Resource UpdateURLs []string @@ -44,6 +45,11 @@ type ResourceRegistry struct { UsePreReleases bool DevMode bool Online bool + + // StateNotifyFunc may be set to receive any changes to the registry state. + // The specified function may lock the state, but may not block or take a + // lot of time. + StateNotifyFunc func(*RegistryState) } // AddIndex adds a new index to the resource registry. @@ -61,6 +67,18 @@ func (reg *ResourceRegistry) AddIndex(idx Index) { reg.indexes = append(reg.indexes, &idx) } +// PreInitUpdateState sets the initial update state of the registry before initialization. +func (reg *ResourceRegistry) PreInitUpdateState(s UpdateState) error { + if reg.state != nil { + return errors.New("registry already initialized") + } + + reg.state = &RegistryState{ + Updates: s, + } + return nil +} + // Initialize initializes a raw registry struct and makes it ready for usage. func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error { // check if storage dir is available @@ -78,6 +96,11 @@ func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error { reg.storageDir = storageDir reg.tmpDir = storageDir.ChildDir("tmp", 0o0700) reg.resources = make(map[string]*Resource) + if reg.state == nil { + reg.state = &RegistryState{} + } + reg.state.ID = StateReady + reg.state.reg = reg // remove tmp dir to delete old entries err = reg.Cleanup() @@ -147,32 +170,34 @@ func (reg *ResourceRegistry) SetUsePreReleases(yes bool) { } // AddResource adds a resource to the registry. Does _not_ select new version. -func (reg *ResourceRegistry) AddResource(identifier, version string, available, currentRelease, preRelease bool) error { +func (reg *ResourceRegistry) AddResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error { reg.Lock() defer reg.Unlock() - err := reg.addResource(identifier, version, available, currentRelease, preRelease) + err := reg.addResource(identifier, version, index, available, currentRelease, preRelease) return err } -func (reg *ResourceRegistry) addResource(identifier, version string, available, currentRelease, preRelease bool) error { +func (reg *ResourceRegistry) addResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error { res, ok := reg.resources[identifier] if !ok { res = reg.newResource(identifier) reg.resources[identifier] = res } + res.Index = index + return res.AddVersion(version, available, currentRelease, preRelease) } // AddResources adds resources to the registry. Errors are logged, the last one is returned. Despite errors, non-failing resources are still added. Does _not_ select new versions. -func (reg *ResourceRegistry) AddResources(versions map[string]string, available, currentRelease, preRelease bool) error { +func (reg *ResourceRegistry) AddResources(versions map[string]string, index *Index, available, currentRelease, preRelease bool) error { reg.Lock() defer reg.Unlock() // add versions and their flags to registry var lastError error for identifier, version := range versions { - lastError = reg.addResource(identifier, version, available, currentRelease, preRelease) + lastError = reg.addResource(identifier, version, index, available, currentRelease, preRelease) if lastError != nil { log.Warningf("%s: failed to add resource %s: %s", reg.Name, identifier, lastError) } diff --git a/updater/resource.go b/updater/resource.go index 8aaa75c..5b94516 100644 --- a/updater/resource.go +++ b/updater/resource.go @@ -55,6 +55,10 @@ type Resource struct { // VerificationOptions holds the verification options for this resource. VerificationOptions *VerificationOptions + + // Index holds a reference to the index this resource was last defined in. + // Will be nil if resource was only found on disk. + Index *Index } // ResourceVersion represents a single version of a resource. @@ -89,7 +93,7 @@ func (rv *ResourceVersion) String() string { return rv.VersionNumber } -// SemVer returns the semantiv version of the resource. +// SemVer returns the semantic version of the resource. func (rv *ResourceVersion) SemVer() *semver.Version { return rv.semVer } diff --git a/updater/state.go b/updater/state.go new file mode 100644 index 0000000..21bc6d1 --- /dev/null +++ b/updater/state.go @@ -0,0 +1,180 @@ +package updater + +import ( + "sort" + "sync" + "time" + + "github.com/safing/portbase/utils" +) + +// Registry States. +const ( + StateReady = "ready" // Default idle state. + StateChecking = "checking" // Downloading indexes. + StateDownloading = "downloading" // Downloading updates. + StateFetching = "fetching" // Fetching a single file. +) + +// RegistryState describes the registry state. +type RegistryState struct { + sync.Mutex + reg *ResourceRegistry + + // ID holds the ID of the state the registry is currently in. + ID string + + // Details holds further information about the current state. + Details any + + // Updates holds generic information about the current status of pending + // and recently downloaded updates. + Updates UpdateState + + // operationLock locks the operation of any state changing operation. + // This is separate from the registry lock, which locks access to the + // registry struct. + operationLock sync.Mutex +} + +// StateDownloadingDetails holds details of the downloading state. +type StateDownloadingDetails struct { + // Resources holds the resource IDs that are being downloaded. + Resources []string + + // FinishedUpTo holds the index of Resources that is currently being + // downloaded. Previous resources have finished downloading. + FinishedUpTo int +} + +// UpdateState holds generic information about the current status of pending +// and recently downloaded updates. +type UpdateState struct { + // LastCheckAt holds the time of the last update check. + LastCheckAt *time.Time + // LastCheckError holds the error of the last check. + LastCheckError error + // PendingDownload holds the resources that are pending download. + PendingDownload []string + + // LastDownloadAt holds the time when resources were downloaded the last time. + LastDownloadAt *time.Time + // LastDownloadError holds the error of the last download. + LastDownloadError error + // LastDownload holds the resources that we downloaded the last time updates + // were downloaded. + LastDownload []string + + // LastSuccessAt holds the time of the last successful update (check). + LastSuccessAt *time.Time +} + +// GetState returns the current registry state. +// The returned data must not be modified. +func (reg *ResourceRegistry) GetState() RegistryState { + reg.state.Lock() + defer reg.state.Unlock() + + return RegistryState{ + ID: reg.state.ID, + Details: reg.state.Details, + Updates: reg.state.Updates, + } +} + +// StartOperation starts an operation. +func (s *RegistryState) StartOperation(id string) bool { + defer s.notify() + + s.operationLock.Lock() + + s.Lock() + defer s.Unlock() + + s.ID = id + return true +} + +// UpdateOperationDetails updates the details of an operation. +// The supplied struct should be a copy and must not be changed after calling +// this function. +func (s *RegistryState) UpdateOperationDetails(details any) { + defer s.notify() + + s.Lock() + defer s.Unlock() + + s.Details = details +} + +// EndOperation ends an operation. +func (s *RegistryState) EndOperation() { + defer s.notify() + defer s.operationLock.Unlock() + + s.Lock() + defer s.Unlock() + + s.ID = StateReady + s.Details = nil +} + +// ReportUpdateCheck reports an update check to the registry state. +func (s *RegistryState) ReportUpdateCheck(pendingDownload []string, failed error) { + defer s.notify() + + sort.Strings(pendingDownload) + + s.Lock() + defer s.Unlock() + + now := time.Now() + s.Updates.LastCheckAt = &now + s.Updates.LastCheckError = failed + s.Updates.PendingDownload = pendingDownload + + if failed == nil { + s.Updates.LastSuccessAt = &now + } +} + +// ReportDownloads reports downloaded updates to the registry state. +func (s *RegistryState) ReportDownloads(downloaded []string, failed error) { + defer s.notify() + + sort.Strings(downloaded) + + s.Lock() + defer s.Unlock() + + now := time.Now() + s.Updates.LastDownloadAt = &now + s.Updates.LastDownloadError = failed + s.Updates.LastDownload = downloaded + + // Remove downloaded resources from the pending list. + if len(s.Updates.PendingDownload) > 0 { + newPendingDownload := make([]string, 0, len(s.Updates.PendingDownload)) + for _, pending := range s.Updates.PendingDownload { + if !utils.StringInSlice(downloaded, pending) { + newPendingDownload = append(newPendingDownload, pending) + } + } + s.Updates.PendingDownload = newPendingDownload + } + + if failed == nil { + s.Updates.LastSuccessAt = &now + } +} + +func (s *RegistryState) notify() { + switch { + case s.reg == nil: + return + case s.reg.StateNotifyFunc == nil: + return + } + + s.reg.StateNotifyFunc(s) +} diff --git a/updater/storage.go b/updater/storage.go index e8ceca1..869b30b 100644 --- a/updater/storage.go +++ b/updater/storage.go @@ -79,7 +79,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error { } // save - err = reg.AddResource(identifier, version, true, false, false) + err = reg.AddResource(identifier, version, nil, true, false, false) if err != nil { lastError = fmt.Errorf("%s: could not get add resource %s v%s: %w", reg.Name, identifier, version, err) log.Warning(lastError.Error()) @@ -178,7 +178,7 @@ func (reg *ResourceRegistry) loadIndexFile(idx *Index) error { } // Add index releases to available resources. - err = reg.AddResources(indexFile.Releases, false, true, idx.PreRelease) + err = reg.AddResources(indexFile.Releases, idx, false, true, idx.PreRelease) if err != nil { log.Warningf("%s: failed to add resource: %s", reg.Name, err) } diff --git a/updater/updating.go b/updater/updating.go index 116db17..0acca37 100644 --- a/updater/updating.go +++ b/updater/updating.go @@ -8,7 +8,8 @@ import ( "path" "path/filepath" "strings" - "sync" + + "golang.org/x/exp/slices" "github.com/safing/jess/filesig" "github.com/safing/jess/lhash" @@ -22,6 +23,10 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error { var lastErr error var anySuccess bool + // Start registry operation. + reg.state.StartOperation(StateChecking) + defer reg.state.EndOperation() + client := &http.Client{} for _, idx := range reg.getIndexes() { if err := reg.downloadIndex(ctx, client, idx); err != nil { @@ -32,9 +37,20 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error { } } + // If all indexes failed to update, fail. if !anySuccess { - return fmt.Errorf("failed to update all indexes, last error was: %w", lastErr) + err := fmt.Errorf("failed to update all indexes, last error was: %w", lastErr) + reg.state.ReportUpdateCheck(nil, err) + return err } + + // Get pending resources and update status. + pendingResourceVersions, _ := reg.GetPendingDownloads(true, false) + reg.state.ReportUpdateCheck( + identifiersFromResourceVersions(pendingResourceVersions), + nil, + ) + return nil } @@ -127,7 +143,7 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli } // add resources to registry - err = reg.AddResources(cleanedData, false, true, idx.PreRelease) + err = reg.AddResources(cleanedData, idx, false, true, idx.PreRelease) if err != nil { log.Warningf("%s: failed to add resources: %s", reg.Name, err) } @@ -167,37 +183,17 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli } // DownloadUpdates checks if updates are available and downloads updates of used components. -func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { - // create list of downloads - var toUpdate []*ResourceVersion - var missingSigs []*ResourceVersion - reg.RLock() - for _, res := range reg.resources { - res.Lock() +func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context, automaticOnly bool) error { + // Start registry operation. + reg.state.StartOperation(StateDownloading) + defer reg.state.EndOperation() - // check if we want to download - if res.inUse() || - res.available() || // resource was used in the past - utils.StringInSlice(reg.MandatoryUpdates, res.Identifier) { // resource is mandatory - - // add all non-available and eligible versions to update queue - for _, rv := range res.Versions { - switch { - case !rv.CurrentRelease: - // We are not interested in older releases. - case !rv.Available: - // File is not available. - toUpdate = append(toUpdate, rv) - case !rv.SigAvailable && res.VerificationOptions != nil: - // File signature is not available and verification is enabled. - missingSigs = append(missingSigs, rv) - } - } - } - - res.Unlock() - } - reg.RUnlock() + // Get pending updates. + toUpdate, missingSigs := reg.GetPendingDownloads(!automaticOnly, true) + downloadDetailsResources := identifiersFromResourceVersions(toUpdate) + reg.state.UpdateOperationDetails(&StateDownloadingDetails{ + Resources: downloadDetailsResources, + }) // nothing to update if len(toUpdate) == 0 && len(missingSigs) == 0 { @@ -211,63 +207,153 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { } // download updates - log.Infof("%s: starting to download %d updates in parallel", reg.Name, len(toUpdate)+len(missingSigs)) - var wg sync.WaitGroup - - wg.Add(len(toUpdate) + len(missingSigs)) + log.Infof("%s: starting to download %d updates", reg.Name, len(toUpdate)) client := &http.Client{} + var reportError error - for idx := range toUpdate { - go func(rv *ResourceVersion) { - var err error - - defer wg.Done() - defer func() { - if x := recover(); x != nil { - log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x) + for i, rv := range toUpdate { + log.Infof( + "%s: downloading update [%d/%d]: %s version %s", + reg.Name, + i+1, len(toUpdate), + rv.resource.Identifier, rv.VersionNumber, + ) + var err error + for tries := 0; tries < 3; tries++ { + err = reg.fetchFile(ctx, client, rv, tries) + if err == nil { + // Update resource version state. + rv.resource.Lock() + rv.Available = true + if rv.resource.VerificationOptions != nil { + rv.SigAvailable = true } - }() + rv.resource.Unlock() - for tries := 0; tries < 3; tries++ { - err = reg.fetchFile(ctx, client, rv, tries) - if err == nil { - rv.Available = true - return - } + break } - if err != nil { - log.Warningf("%s: failed to download %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err) - } - }(toUpdate[idx]) + } + if err != nil { + reportError := fmt.Errorf("failed to download %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err) + log.Warningf("%s: %s", reg.Name, reportError) + } + + reg.state.UpdateOperationDetails(&StateDownloadingDetails{ + Resources: downloadDetailsResources, + FinishedUpTo: i + 1, + }) } - for idx := range missingSigs { - go func(rv *ResourceVersion) { + if len(missingSigs) > 0 { + log.Infof("%s: downloading %d missing signatures", reg.Name, len(missingSigs)) + + for _, rv := range missingSigs { var err error - - defer wg.Done() - defer func() { - if x := recover(); x != nil { - log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x) - } - }() - for tries := 0; tries < 3; tries++ { err = reg.fetchMissingSig(ctx, client, rv, tries) if err == nil { + // Update resource version state. + rv.resource.Lock() rv.SigAvailable = true - return + rv.resource.Unlock() + + break } } if err != nil { - log.Warningf("%s: failed to download missing sig of %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err) + reportError := fmt.Errorf("failed to download missing sig of %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err) + log.Warningf("%s: %s", reg.Name, reportError) } - }(missingSigs[idx]) + } } - wg.Wait() - + reg.state.ReportDownloads( + downloadDetailsResources, + reportError, + ) log.Infof("%s: finished downloading updates", reg.Name) return nil } + +// DownloadUpdates checks if updates are available and downloads updates of used components. + +// GetPendingDownloads returns the list of pending downloads. +// If manual is set, indexes with AutoDownload=false will be checked. +// If auto is set, indexes with AutoDownload=true will be checked. +func (reg *ResourceRegistry) GetPendingDownloads(manual, auto bool) (resources, sigs []*ResourceVersion) { + reg.RLock() + defer reg.RUnlock() + + // create list of downloads + var toUpdate []*ResourceVersion + var missingSigs []*ResourceVersion + + for _, res := range reg.resources { + func() { + res.Lock() + defer res.Unlock() + + // Skip resources without index or indexes that should not be reported + // according to parameters. + switch { + case res.Index == nil: + // Cannot download if resource is not part of an index. + return + case manual && !res.Index.AutoDownload: + // Manual update report and index is not auto-download. + case auto && res.Index.AutoDownload: + // Auto update report and index is auto-download. + default: + // Resource should not be reported. + return + } + + // Skip resources we don't need. + switch { + case res.inUse(): + // Update if resource is in use. + case res.available(): + // Update if resource is available locally, ie. was used in the past. + case utils.StringInSlice(reg.MandatoryUpdates, res.Identifier): + // Update is set as mandatory. + default: + // Resource does not need to be updated. + return + } + + // Go through all versions until we find versions that need updating. + for _, rv := range res.Versions { + switch { + case !rv.CurrentRelease: + // We are not interested in older releases. + case !rv.Available: + // File not available locally, download! + toUpdate = append(toUpdate, rv) + case !rv.SigAvailable && res.VerificationOptions != nil: + // File signature is not available and verification is enabled, download signature! + missingSigs = append(missingSigs, rv) + } + } + }() + } + + slices.SortFunc[*ResourceVersion](toUpdate, func(a, b *ResourceVersion) bool { + return a.resource.Identifier < b.resource.Identifier + }) + slices.SortFunc[*ResourceVersion](missingSigs, func(a, b *ResourceVersion) bool { + return a.resource.Identifier < b.resource.Identifier + }) + + return toUpdate, missingSigs +} + +func identifiersFromResourceVersions(resourceVersions []*ResourceVersion) []string { + identifiers := make([]string, len(resourceVersions)) + + for i, rv := range resourceVersions { + identifiers[i] = rv.resource.Identifier + } + + return identifiers +} diff --git a/utils/debug/debug.go b/utils/debug/debug.go index 7ff5dec..70e5192 100644 --- a/utils/debug/debug.go +++ b/utils/debug/debug.go @@ -2,14 +2,11 @@ package debug import ( "bytes" - "context" "fmt" "runtime/pprof" "strings" "time" - "github.com/shirou/gopsutil/host" - "github.com/safing/portbase/info" "github.com/safing/portbase/log" "github.com/safing/portbase/modules" @@ -94,39 +91,6 @@ func (di *Info) AddVersionInfo() { ) } -// AddPlatformInfo adds OS and platform information. -func (di *Info) AddPlatformInfo(ctx context.Context) { - // Get information from the system. - info, err := host.InfoWithContext(ctx) - if err != nil { - di.AddSection( - "Platform Information", - NoFlags, - fmt.Sprintf("Failed to get: %s", err), - ) - return - } - - // Check if we want to add virtulization information. - var virtInfo string - if info.VirtualizationRole == "guest" { - if info.VirtualizationSystem != "" { - virtInfo = fmt.Sprintf("VM: %s", info.VirtualizationSystem) - } else { - virtInfo = "VM: unidentified" - } - } - - // Add section. - di.AddSection( - fmt.Sprintf("Platform: %s %s", info.Platform, info.PlatformVersion), - UseCodeSection|AddContentLineBreaks, - fmt.Sprintf("System: %s %s (%s) %s", info.Platform, info.OS, info.PlatformFamily, info.PlatformVersion), - fmt.Sprintf("Kernel: %s %s", info.KernelVersion, info.KernelArch), - virtInfo, - ) -} - // AddGoroutineStack adds the current goroutine stack. func (di *Info) AddGoroutineStack() { buf := new(bytes.Buffer) diff --git a/utils/debug/debug_android.go b/utils/debug/debug_android.go new file mode 100644 index 0000000..c5df94b --- /dev/null +++ b/utils/debug/debug_android.go @@ -0,0 +1,31 @@ +package debug + +import ( + "context" + "fmt" + + "github.com/safing/portmaster-android/go/app_interface" +) + +// AddPlatformInfo adds OS and platform information. +func (di *Info) AddPlatformInfo(_ context.Context) { + // Get information from the system. + info, err := app_interface.GetPlatformInfo() + if err != nil { + di.AddSection( + "Platform Information", + NoFlags, + fmt.Sprintf("Failed to get: %s", err), + ) + return + } + + // Add section. + di.AddSection( + fmt.Sprintf("Platform: Android"), + UseCodeSection|AddContentLineBreaks, + fmt.Sprintf("SDK: %d", info.SDK), + fmt.Sprintf("Device: %s %s (%s)", info.Manufacturer, info.Brand, info.Board), + fmt.Sprintf("App: %s: %s(%d) %s", info.ApplicationID, info.VersionName, info.VersionCode, info.BuildType)) + +} diff --git a/utils/debug/debug_default.go b/utils/debug/debug_default.go new file mode 100644 index 0000000..cb429b8 --- /dev/null +++ b/utils/debug/debug_default.go @@ -0,0 +1,43 @@ +//go:build !android + +package debug + +import ( + "context" + "fmt" + + "github.com/shirou/gopsutil/host" +) + +// AddPlatformInfo adds OS and platform information. +func (di *Info) AddPlatformInfo(ctx context.Context) { + // Get information from the system. + info, err := host.InfoWithContext(ctx) + if err != nil { + di.AddSection( + "Platform Information", + NoFlags, + fmt.Sprintf("Failed to get: %s", err), + ) + return + } + + // Check if we want to add virtulization information. + var virtInfo string + if info.VirtualizationRole == "guest" { + if info.VirtualizationSystem != "" { + virtInfo = fmt.Sprintf("VM: %s", info.VirtualizationSystem) + } else { + virtInfo = "VM: unidentified" + } + } + + // Add section. + di.AddSection( + fmt.Sprintf("Platform: %s %s", info.Platform, info.PlatformVersion), + UseCodeSection|AddContentLineBreaks, + fmt.Sprintf("System: %s %s (%s) %s", info.Platform, info.OS, info.PlatformFamily, info.PlatformVersion), + fmt.Sprintf("Kernel: %s %s", info.KernelVersion, info.KernelArch), + virtInfo, + ) +}