Add and expose updater registry state

go.mod

@@ -1,6 +1,6 @@
 module github.com/safing/portbase
 
-go 1.15
+go 1.19
 
 require (
 	github.com/VictoriaMetrics/metrics v1.22.2

updater/get.go

@@ -52,6 +52,10 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
 		return nil, fmt.Errorf("could not prepare tmp directory for download: %w", err)
 	}
 
+	// Start registry operation.
+	reg.state.StartOperation(StateFetching)
+	defer reg.state.EndOperation()
+
 	// download file
 	log.Tracef("%s: starting download of %s", reg.Name, file.versionedPath)
 	client := &http.Client{}
@@ -69,3 +73,19 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
 	log.Warningf("%s: failed to download %s: %s", reg.Name, file.versionedPath, err)
 	return nil, err
 }
+
+// GetVersion returns the selected version of the given identifier.
+// The returned resource version may not be modified.
+func (reg *ResourceRegistry) GetVersion(identifier string) (*ResourceVersion, error) {
+	reg.RLock()
+	res, ok := reg.resources[identifier]
+	reg.RUnlock()
+	if !ok {
+		return nil, ErrNotFound
+	}
+
+	res.Lock()
+	defer res.Unlock()
+
+	return res.SelectedVersion, nil
+}

updater/indexes.go

@@ -27,6 +27,9 @@ type Index struct {
 	// not.
 	PreRelease bool
 
+	// AutoDownload specifies whether new versions should be automatically downloaded.
+	AutoDownload bool
+
 	// LastRelease holds the time of the last seen release of this index.
 	LastRelease time.Time
 }

updater/registry.go

@@ -25,6 +25,7 @@ type ResourceRegistry struct {
 	storageDir *utils.DirStructure
 	tmpDir     *utils.DirStructure
 	indexes    []*Index
+	state      *RegistryState
 
 	resources        map[string]*Resource
 	UpdateURLs       []string
@@ -44,6 +45,11 @@ type ResourceRegistry struct {
 	UsePreReleases bool
 	DevMode        bool
 	Online         bool
+
+	// StateNotifyFunc may be set to receive any changes to the registry state.
+	// The specified function may lock the state, but may not block are take a
+	// lot of time.
+	StateNotifyFunc func(*RegistryState)
 }
 
 // AddIndex adds a new index to the resource registry.
@@ -61,6 +67,18 @@ func (reg *ResourceRegistry) AddIndex(idx Index) {
 	reg.indexes = append(reg.indexes, &idx)
 }
 
+// PreInitUpdateState sets the initial update state of the registry before initialization.
+func (reg *ResourceRegistry) PreInitUpdateState(s UpdateState) error {
+	if reg.state != nil {
+		return errors.New("registry already initialized")
+	}
+
+	reg.state = &RegistryState{
+		Updates: s,
+	}
+	return nil
+}
+
 // Initialize initializes a raw registry struct and makes it ready for usage.
 func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
 	// check if storage dir is available
@@ -78,6 +96,11 @@ func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
 	reg.storageDir = storageDir
 	reg.tmpDir = storageDir.ChildDir("tmp", 0o0700)
 	reg.resources = make(map[string]*Resource)
+	if reg.state == nil {
+		reg.state = &RegistryState{}
+	}
+	reg.state.ID = StateReady
+	reg.state.reg = reg
 
 	// remove tmp dir to delete old entries
 	err = reg.Cleanup()
@@ -147,32 +170,34 @@ func (reg *ResourceRegistry) SetUsePreReleases(yes bool) {
 }
 
 // AddResource adds a resource to the registry. Does _not_ select new version.
-func (reg *ResourceRegistry) AddResource(identifier, version string, available, currentRelease, preRelease bool) error {
+func (reg *ResourceRegistry) AddResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error {
 	reg.Lock()
 	defer reg.Unlock()
 
-	err := reg.addResource(identifier, version, available, currentRelease, preRelease)
+	err := reg.addResource(identifier, version, index, available, currentRelease, preRelease)
 	return err
 }
 
-func (reg *ResourceRegistry) addResource(identifier, version string, available, currentRelease, preRelease bool) error {
+func (reg *ResourceRegistry) addResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error {
 	res, ok := reg.resources[identifier]
 	if !ok {
 		res = reg.newResource(identifier)
 		reg.resources[identifier] = res
 	}
+	res.Index = index
+
 	return res.AddVersion(version, available, currentRelease, preRelease)
 }
 
 // AddResources adds resources to the registry. Errors are logged, the last one is returned. Despite errors, non-failing resources are still added. Does _not_ select new versions.
-func (reg *ResourceRegistry) AddResources(versions map[string]string, available, currentRelease, preRelease bool) error {
+func (reg *ResourceRegistry) AddResources(versions map[string]string, index *Index, available, currentRelease, preRelease bool) error {
 	reg.Lock()
 	defer reg.Unlock()
 
 	// add versions and their flags to registry
 	var lastError error
 	for identifier, version := range versions {
-		lastError = reg.addResource(identifier, version, available, currentRelease, preRelease)
+		lastError = reg.addResource(identifier, version, index, available, currentRelease, preRelease)
 		if lastError != nil {
 			log.Warningf("%s: failed to add resource %s: %s", reg.Name, identifier, lastError)
 		}

updater/resource.go

@@ -55,6 +55,10 @@ type Resource struct {
 
 	// VerificationOptions holds the verification options for this resource.
 	VerificationOptions *VerificationOptions
+
+	// Index holds a reference to the index this resource was last defined in.
+	// Will be nil if resource was only found on disk.
+	Index *Index
 }
 
 // ResourceVersion represents a single version of a resource.

updater/state.go

@@ -0,0 +1,165 @@
+package updater
+
+import (
+	"sync"
+	"time"
+)
+
+// Registry States.
+const (
+	StateReady       = "ready"       // Default idle state.
+	StateChecking    = "checking"    // Downloading indexes.
+	StateDownloading = "downloading" // Downloading updates.
+	StateFetching    = "fetching"    // Fetching a single file.
+)
+
+// RegistryState describtes the registry state.
+type RegistryState struct {
+	sync.Mutex
+	reg *ResourceRegistry
+
+	// ID holds the ID of the state the registry is currently in.
+	ID string
+
+	// Details holds further information about the current state.
+	Details any
+
+	// Updates holds generic information about the current status of pending
+	// and recently downloaded updates.
+	Updates UpdateState
+
+	// operationLock locks the operation of any state changing operation.
+	// This is separate from the registry lock, which locks access to the
+	// registry struct.
+	operationLock sync.Mutex
+}
+
+// StateDownloadingDetails holds details of the downloading state.
+type StateDownloadingDetails struct {
+	// Resources holds the resource IDs that are being downloaded.
+	Resources []string
+
+	// FinishedUpTo holds the index of Resources that is currently being
+	// downloaded. Previous resources have finished downloading.
+	FinishedUpTo int
+}
+
+// UpdateState holds generic information about the current status of pending
+// and recently downloaded updates.
+type UpdateState struct {
+	// LastCheckAt holds the time of the last update check.
+	LastCheckAt *time.Time
+	// LastCheckError holds the error of the last check.
+	LastCheckError error
+	// PendingDownload holds the resources that are pending download.
+	PendingDownload []string
+
+	// LastDownloadAt holds the time when resources were downloaded the last time.
+	LastDownloadAt *time.Time
+	// LastDownloadError holds the error of the last download.
+	LastDownloadError error
+	// LastDownload holds the resources that we downloaded the last time udpates
+	// were downloaded.
+	LastDownload []string
+
+	// LastSuccessAt holds the time of the last successful update (check).
+	LastSuccessAt *time.Time
+}
+
+// GetState returns the current registry state.
+// The returned data must not be modified.
+func (reg *ResourceRegistry) GetState() RegistryState {
+	reg.state.Lock()
+	defer reg.state.Unlock()
+
+	return RegistryState{
+		ID:      reg.state.ID,
+		Details: reg.state.Details,
+		Updates: reg.state.Updates,
+	}
+}
+
+// StartOperation starts an operation.
+func (s *RegistryState) StartOperation(id string) bool {
+	defer s.notify()
+
+	s.operationLock.Lock()
+
+	s.Lock()
+	defer s.Unlock()
+
+	s.ID = id
+	return true
+}
+
+// UpdateOperationDetails updates the details of an operation.
+// The supplied struct should be a copy and must not be changed after calling
+// this function.
+func (s *RegistryState) UpdateOperationDetails(details any) {
+	defer s.notify()
+
+	s.Lock()
+	defer s.Unlock()
+
+	s.Details = details
+}
+
+// EndOperation ends an operation.
+func (s *RegistryState) EndOperation() {
+	defer s.notify()
+	defer s.operationLock.Unlock()
+
+	s.Lock()
+	defer s.Unlock()
+
+	s.ID = StateReady
+	s.Details = nil
+}
+
+// ReportUpdateCheck reports an update check to the registry state.
+func (s *RegistryState) ReportUpdateCheck(pendingDownload []string, failed error) {
+	defer s.notify()
+
+	s.Lock()
+	defer s.Unlock()
+
+	now := time.Now()
+	s.Updates.LastCheckAt = &now
+	s.Updates.LastCheckError = failed
+	s.Updates.PendingDownload = pendingDownload
+
+	if failed == nil {
+		s.Updates.LastSuccessAt = &now
+	}
+}
+
+// ReportDownloads reports downloaded updates to the registry state.
+func (s *RegistryState) ReportDownloads(downloaded []string, failed error) {
+	defer s.notify()
+
+	s.Lock()
+	defer s.Unlock()
+
+	now := time.Now()
+	s.Updates.LastDownloadAt = &now
+	s.Updates.LastDownloadError = failed
+	s.Updates.LastDownload = downloaded
+
+	// Reset pending downloads, as they have now been downloaded.
+	s.Updates.PendingDownload = nil
+
+	if failed == nil {
+		s.Updates.LastSuccessAt = &now
+	}
+}
+
+func (s *RegistryState) notify() {
+	switch {
+	case s.reg == nil:
+		return
+	case s.reg.StateNotifyFunc == nil:
+		return
+	}
+
+	s.reg.StateNotifyFunc(s)
+}

updater/storage.go

@@ -79,7 +79,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error {
 		}
 
 		// save
-		err = reg.AddResource(identifier, version, true, false, false)
+		err = reg.AddResource(identifier, version, nil, true, false, false)
 		if err != nil {
 			lastError = fmt.Errorf("%s: could not get add resource %s v%s: %w", reg.Name, identifier, version, err)
 			log.Warning(lastError.Error())
@@ -178,7 +178,7 @@ func (reg *ResourceRegistry) loadIndexFile(idx *Index) error {
 	}
 
 	// Add index releases to available resources.
-	err = reg.AddResources(indexFile.Releases, false, true, idx.PreRelease)
+	err = reg.AddResources(indexFile.Releases, idx, false, true, idx.PreRelease)
 	if err != nil {
 		log.Warningf("%s: failed to add resource: %s", reg.Name, err)
 	}

updater/updating.go

@@ -8,7 +8,6 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
-	"sync"
 
 	"github.com/safing/jess/filesig"
 	"github.com/safing/jess/lhash"
@@ -22,6 +21,10 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
 	var lastErr error
 	var anySuccess bool
 
+	// Start registry operation.
+	reg.state.StartOperation(StateChecking)
+	defer reg.state.EndOperation()
+
 	client := &http.Client{}
 	for _, idx := range reg.getIndexes() {
 		if err := reg.downloadIndex(ctx, client, idx); err != nil {
@@ -32,9 +35,20 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
 		}
 	}
 
+	// If all indexes failed to update, fail.
 	if !anySuccess {
-		return fmt.Errorf("failed to update all indexes, last error was: %w", lastErr)
+		err := fmt.Errorf("failed to update all indexes, last error was: %w", lastErr)
+		reg.state.ReportUpdateCheck(nil, err)
+		return err
 	}
+
+	// Get pending resources and update status.
+	pendingResourceVersions, _ := reg.GetPendingDownloads(true, false)
+	reg.state.ReportUpdateCheck(
+		identifiersFromResourceVersions(pendingResourceVersions),
+		nil,
+	)
+
 	return nil
 }
 
@@ -127,7 +141,7 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
 		}
 
 		// add resources to registry
-		err = reg.AddResources(cleanedData, false, true, idx.PreRelease)
+		err = reg.AddResources(cleanedData, idx, false, true, idx.PreRelease)
 		if err != nil {
 			log.Warningf("%s: failed to add resources: %s", reg.Name, err)
 		}
@@ -167,37 +181,17 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
 }
 
 // DownloadUpdates checks if updates are available and downloads updates of used components.
-func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
+func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context, automaticOnly bool) error {
-	// create list of downloads
+	// Start registry operation.
-	var toUpdate []*ResourceVersion
+	reg.state.StartOperation(StateDownloading)
-	var missingSigs []*ResourceVersion
+	defer reg.state.EndOperation()
-	reg.RLock()
-	for _, res := range reg.resources {
-		res.Lock()
 
-		// check if we want to download
+	// Get pending updates.
-		if res.inUse() ||
+	toUpdate, missingSigs := reg.GetPendingDownloads(!automaticOnly, true)
-			res.available() || // resource was used in the past
+	downloadDetailsResources := identifiersFromResourceVersions(toUpdate)
-			utils.StringInSlice(reg.MandatoryUpdates, res.Identifier) { // resource is mandatory
+	reg.state.UpdateOperationDetails(&StateDownloadingDetails{
-
+		Resources: downloadDetailsResources,
-			// add all non-available and eligible versions to update queue
+	})
-			for _, rv := range res.Versions {
-				switch {
-				case !rv.CurrentRelease:
-					// We are not interested in older releases.
-				case !rv.Available:
-					// File is not available.
-					toUpdate = append(toUpdate, rv)
-				case !rv.SigAvailable && res.VerificationOptions != nil:
-					// File signature is not available and verification is enabled.
-					missingSigs = append(missingSigs, rv)
-				}
-			}
-		}
-
-		res.Unlock()
-	}
-	reg.RUnlock()
 
 	// nothing to update
 	if len(toUpdate) == 0 && len(missingSigs) == 0 {
@@ -211,63 +205,136 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
 	}
 
 	// download updates
-	log.Infof("%s: starting to download %d updates in parallel", reg.Name, len(toUpdate)+len(missingSigs))
+	log.Infof("%s: starting to download %d updates", reg.Name, len(toUpdate)+len(missingSigs))
-	var wg sync.WaitGroup
-
-	wg.Add(len(toUpdate) + len(missingSigs))
 	client := &http.Client{}
+	var reportError error
 
-	for idx := range toUpdate {
+	for i, rv := range toUpdate {
-		go func(rv *ResourceVersion) {
+		var err error
-			var err error
+		for tries := 0; tries < 3; tries++ {
-
+			err = reg.fetchFile(ctx, client, rv, tries)
-			defer wg.Done()
+			if err == nil {
-			defer func() {
+				// Update resource version state.
-				if x := recover(); x != nil {
+				rv.resource.Lock()
-					log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x)
+				rv.Available = true
-				}
+				if rv.resource.VerificationOptions != nil {
-			}()
-
-			for tries := 0; tries < 3; tries++ {
-				err = reg.fetchFile(ctx, client, rv, tries)
-				if err == nil {
-					rv.Available = true
-					return
-				}
-			}
-			if err != nil {
-				log.Warningf("%s: failed to download %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
-			}
-		}(toUpdate[idx])
-	}
-
-	for idx := range missingSigs {
-		go func(rv *ResourceVersion) {
-			var err error
-
-			defer wg.Done()
-			defer func() {
-				if x := recover(); x != nil {
-					log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x)
-				}
-			}()
-
-			for tries := 0; tries < 3; tries++ {
-				err = reg.fetchMissingSig(ctx, client, rv, tries)
-				if err == nil {
 					rv.SigAvailable = true
-					return
 				}
+				rv.resource.Unlock()
+
+				break
 			}
-			if err != nil {
+		}
-				log.Warningf("%s: failed to download missing sig of %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
+		if err != nil {
-			}
+			reportError := fmt.Errorf("failed to download %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
-		}(missingSigs[idx])
+			log.Warningf("%s: %s", reg.Name, reportError)
+		}
+
+		reg.state.UpdateOperationDetails(&StateDownloadingDetails{
+			Resources:    downloadDetailsResources,
+			FinishedUpTo: i + 1,
+		})
 	}
 
-	wg.Wait()
+	for _, rv := range missingSigs {
+		var err error
+		for tries := 0; tries < 3; tries++ {
+			err = reg.fetchMissingSig(ctx, client, rv, tries)
+			if err == nil {
+				// Update resource version state.
+				rv.resource.Lock()
+				rv.SigAvailable = true
+				rv.resource.Unlock()
 
+				break
+			}
+		}
+		if err != nil {
+			reportError := fmt.Errorf("failed to download missing sig of %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
+			log.Warningf("%s: %s", reg.Name, reportError)
+		}
+	}
+
+	reg.state.ReportDownloads(
+		downloadDetailsResources,
+		reportError,
+	)
 	log.Infof("%s: finished downloading updates", reg.Name)
 
 	return nil
 }
+
+// DownloadUpdates checks if updates are available and downloads updates of used components.
+
+// GetPendingDownloads returns the list of pending downloads.
+// If manual is set, indexes with AutoDownload=false will be checked.
+// If auto is set, indexes with AutoDownload=true will be checked.
+func (reg *ResourceRegistry) GetPendingDownloads(manual, auto bool) (resources, sigs []*ResourceVersion) {
+	reg.RLock()
+	defer reg.RUnlock()
+
+	// create list of downloads
+	var toUpdate []*ResourceVersion
+	var missingSigs []*ResourceVersion
+
+	for _, res := range reg.resources {
+		func() {
+			res.Lock()
+			defer res.Unlock()
+
+			// Skip resources without index or indexes that should not be reported
+			// according to parameters.
+			switch {
+			case res.Index == nil:
+				// Cannot download if resource is not part of an index.
+				return
+			case manual && !res.Index.AutoDownload:
+				// Manual update report and index is not auto-download.
+			case auto && res.Index.AutoDownload:
+				// Auto update report and index is auto-download.
+			default:
+				// Resource should not be reported.
+				return
+			}
+
+			// Skip resources we don't need.
+			switch {
+			case res.inUse():
+				// Update if resource is in use.
+			case res.available():
+				// Update if resource is available locally, ie. was used in the past.
+			case utils.StringInSlice(reg.MandatoryUpdates, res.Identifier):
+				// Update is set as mandatory.
+			default:
+				// Resource does not need to be updated.
+				return
+			}
+
+			// Go through all versions until we find versions that need updating.
+			for _, rv := range res.Versions {
+				switch {
+				case !rv.CurrentRelease:
+					// We are not interested in older releases.
+				case !rv.Available:
+					// File not available locally, download!
+					toUpdate = append(toUpdate, rv)
+				case !rv.SigAvailable && res.VerificationOptions != nil:
+					// File signature is not available and verification is enabled, download signature!
+					missingSigs = append(missingSigs, rv)
+				}
+			}
+		}()
+	}
+
+	return toUpdate, missingSigs
+}
+
+func identifiersFromResourceVersions(resourceVersions []*ResourceVersion) []string {
+	identifiers := make([]string, len(resourceVersions))
+
+	for i, rv := range resourceVersions {
+		identifiers[i] = rv.resource.Identifier
+	}
+
+	return identifiers
+}