vikarti.anatra/safing-portbase
1
0
Fork 0
mirror of https://github.com/safing/portbase synced 2025-09-01 10:09:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add and expose updater registry state

Browse source
This commit is contained in:
Daniel 2023-03-09 12:05:59 +01:00
parent 0d13bca496
commit d6687ecbad
8 changed files with 373 additions and 89 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
go.mod
View file

@ -1,6 +1,6 @@
module github.com/safing/portbase
go 1.15
go 1.19
require (
github.com/VictoriaMetrics/metrics v1.22.2

20
updater/get.go
View file

@ -52,6 +52,10 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
return nil, fmt.Errorf("could not prepare tmp directory for download: %w", err)
}
// Start registry operation.
reg.state.StartOperation(StateFetching)
defer reg.state.EndOperation()
// download file
log.Tracef("%s: starting download of %s", reg.Name, file.versionedPath)
client := &http.Client{}
@ -69,3 +73,19 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
log.Warningf("%s: failed to download %s: %s", reg.Name, file.versionedPath, err)
return nil, err
}
// GetVersion returns the selected version of the given identifier.
// The returned resource version may not be modified.
func (reg *ResourceRegistry) GetVersion(identifier string) (*ResourceVersion, error) {
reg.RLock()
res, ok := reg.resources[identifier]
reg.RUnlock()
if !ok {
return nil, ErrNotFound
}
res.Lock()
defer res.Unlock()
return res.SelectedVersion, nil
}

3
updater/indexes.go
View file

@ -27,6 +27,9 @@ type Index struct {
// not.
PreRelease bool
// AutoDownload specifies whether new versions should be automatically downloaded.
AutoDownload bool
// LastRelease holds the time of the last seen release of this index.
LastRelease time.Time
}

35
updater/registry.go
View file

@ -25,6 +25,7 @@ type ResourceRegistry struct {
storageDir *utils.DirStructure
tmpDir *utils.DirStructure
indexes []*Index
state *RegistryState
resources map[string]*Resource
UpdateURLs []string
@ -44,6 +45,11 @@ type ResourceRegistry struct {
UsePreReleases bool
DevMode bool
Online bool
// StateNotifyFunc may be set to receive any changes to the registry state.
// The specified function may lock the state, but may not block are take a
// lot of time.
StateNotifyFunc func(*RegistryState)
}
// AddIndex adds a new index to the resource registry.
@ -61,6 +67,18 @@ func (reg *ResourceRegistry) AddIndex(idx Index) {
reg.indexes = append(reg.indexes, &idx)
}
// PreInitUpdateState sets the initial update state of the registry before initialization.
func (reg *ResourceRegistry) PreInitUpdateState(s UpdateState) error {
if reg.state != nil {
return errors.New("registry already initialized")
}
reg.state = &RegistryState{
Updates: s,
}
return nil
}
// Initialize initializes a raw registry struct and makes it ready for usage.
func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
// check if storage dir is available
@ -78,6 +96,11 @@ func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
reg.storageDir = storageDir
reg.tmpDir = storageDir.ChildDir("tmp", 0o0700)
reg.resources = make(map[string]*Resource)
if reg.state == nil {
reg.state = &RegistryState{}
}
reg.state.ID = StateReady
reg.state.reg = reg
// remove tmp dir to delete old entries
err = reg.Cleanup()
@ -147,32 +170,34 @@ func (reg *ResourceRegistry) SetUsePreReleases(yes bool) {
}
// AddResource adds a resource to the registry. Does _not_ select new version.
func (reg *ResourceRegistry) AddResource(identifier, version string, available, currentRelease, preRelease bool) error {
func (reg *ResourceRegistry) AddResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error {
reg.Lock()
defer reg.Unlock()
err := reg.addResource(identifier, version, available, currentRelease, preRelease)
err := reg.addResource(identifier, version, index, available, currentRelease, preRelease)
return err
}
func (reg *ResourceRegistry) addResource(identifier, version string, available, currentRelease, preRelease bool) error {
func (reg *ResourceRegistry) addResource(identifier, version string, index *Index, available, currentRelease, preRelease bool) error {
res, ok := reg.resources[identifier]
if !ok {
res = reg.newResource(identifier)
reg.resources[identifier] = res
}
res.Index = index
return res.AddVersion(version, available, currentRelease, preRelease)
}
// AddResources adds resources to the registry. Errors are logged, the last one is returned. Despite errors, non-failing resources are still added. Does _not_ select new versions.
func (reg *ResourceRegistry) AddResources(versions map[string]string, available, currentRelease, preRelease bool) error {
func (reg *ResourceRegistry) AddResources(versions map[string]string, index *Index, available, currentRelease, preRelease bool) error {
reg.Lock()
defer reg.Unlock()
// add versions and their flags to registry
var lastError error
for identifier, version := range versions {
lastError = reg.addResource(identifier, version, available, currentRelease, preRelease)
lastError = reg.addResource(identifier, version, index, available, currentRelease, preRelease)
if lastError != nil {
log.Warningf("%s: failed to add resource %s: %s", reg.Name, identifier, lastError)
}

4
updater/resource.go
View file

@ -55,6 +55,10 @@ type Resource struct {
// VerificationOptions holds the verification options for this resource.
VerificationOptions *VerificationOptions
// Index holds a reference to the index this resource was last defined in.
// Will be nil if resource was only found on disk.
Index *Index
}
// ResourceVersion represents a single version of a resource.

165
updater/state.go Normal file
View file

@ -0,0 +1,165 @@
package updater
import (
"sync"
"time"
)
// Registry States.
const (
StateReady = "ready" // Default idle state.
StateChecking = "checking" // Downloading indexes.
StateDownloading = "downloading" // Downloading updates.
StateFetching = "fetching" // Fetching a single file.
)
// RegistryState describtes the registry state.
type RegistryState struct {
sync.Mutex
reg *ResourceRegistry
// ID holds the ID of the state the registry is currently in.
ID string
// Details holds further information about the current state.
Details any
// Updates holds generic information about the current status of pending
// and recently downloaded updates.
Updates UpdateState
// operationLock locks the operation of any state changing operation.
// This is separate from the registry lock, which locks access to the
// registry struct.
operationLock sync.Mutex
}
// StateDownloadingDetails holds details of the downloading state.
type StateDownloadingDetails struct {
// Resources holds the resource IDs that are being downloaded.
Resources []string
// FinishedUpTo holds the index of Resources that is currently being
// downloaded. Previous resources have finished downloading.
FinishedUpTo int
}
// UpdateState holds generic information about the current status of pending
// and recently downloaded updates.
type UpdateState struct {
// LastCheckAt holds the time of the last update check.
LastCheckAt *time.Time
// LastCheckError holds the error of the last check.
LastCheckError error
// PendingDownload holds the resources that are pending download.
PendingDownload []string
// LastDownloadAt holds the time when resources were downloaded the last time.
LastDownloadAt *time.Time
// LastDownloadError holds the error of the last download.
LastDownloadError error
// LastDownload holds the resources that we downloaded the last time udpates
// were downloaded.
LastDownload []string
// LastSuccessAt holds the time of the last successful update (check).
LastSuccessAt *time.Time
}
// GetState returns the current registry state.
// The returned data must not be modified.
func (reg *ResourceRegistry) GetState() RegistryState {
reg.state.Lock()
defer reg.state.Unlock()
return RegistryState{
ID: reg.state.ID,
Details: reg.state.Details,
Updates: reg.state.Updates,
}
}
// StartOperation starts an operation.
func (s *RegistryState) StartOperation(id string) bool {
defer s.notify()
s.operationLock.Lock()
s.Lock()
defer s.Unlock()
s.ID = id
return true
}
// UpdateOperationDetails updates the details of an operation.
// The supplied struct should be a copy and must not be changed after calling
// this function.
func (s *RegistryState) UpdateOperationDetails(details any) {
defer s.notify()
s.Lock()
defer s.Unlock()
s.Details = details
}
// EndOperation ends an operation.
func (s *RegistryState) EndOperation() {
defer s.notify()
defer s.operationLock.Unlock()
s.Lock()
defer s.Unlock()
s.ID = StateReady
s.Details = nil
}
// ReportUpdateCheck reports an update check to the registry state.
func (s *RegistryState) ReportUpdateCheck(pendingDownload []string, failed error) {
defer s.notify()
s.Lock()
defer s.Unlock()
now := time.Now()
s.Updates.LastCheckAt = &now
s.Updates.LastCheckError = failed
s.Updates.PendingDownload = pendingDownload
if failed == nil {
s.Updates.LastSuccessAt = &now
}
}
// ReportDownloads reports downloaded updates to the registry state.
func (s *RegistryState) ReportDownloads(downloaded []string, failed error) {
defer s.notify()
s.Lock()
defer s.Unlock()
now := time.Now()
s.Updates.LastDownloadAt = &now
s.Updates.LastDownloadError = failed
s.Updates.LastDownload = downloaded
// Reset pending downloads, as they have now been downloaded.
s.Updates.PendingDownload = nil
if failed == nil {
s.Updates.LastSuccessAt = &now
}
}
func (s *RegistryState) notify() {
switch {
case s.reg == nil:
return
case s.reg.StateNotifyFunc == nil:
return
}
s.reg.StateNotifyFunc(s)
}

4
updater/storage.go
View file

@ -79,7 +79,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error {
}
// save
err = reg.AddResource(identifier, version, true, false, false)
err = reg.AddResource(identifier, version, nil, true, false, false)
if err != nil {
lastError = fmt.Errorf("%s: could not get add resource %s v%s: %w", reg.Name, identifier, version, err)
log.Warning(lastError.Error())
@ -178,7 +178,7 @@ func (reg *ResourceRegistry) loadIndexFile(idx *Index) error {
}
// Add index releases to available resources.
err = reg.AddResources(indexFile.Releases, false, true, idx.PreRelease)
err = reg.AddResources(indexFile.Releases, idx, false, true, idx.PreRelease)
if err != nil {
log.Warningf("%s: failed to add resource: %s", reg.Name, err)
}

229
updater/updating.go
View file

@ -8,7 +8,6 @@ import (
"path"
"path/filepath"
"strings"
"sync"
"github.com/safing/jess/filesig"
"github.com/safing/jess/lhash"
@ -22,6 +21,10 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
var lastErr error
var anySuccess bool
// Start registry operation.
reg.state.StartOperation(StateChecking)
defer reg.state.EndOperation()
client := &http.Client{}
for _, idx := range reg.getIndexes() {
if err := reg.downloadIndex(ctx, client, idx); err != nil {
@ -32,9 +35,20 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
}
}
// If all indexes failed to update, fail.
if !anySuccess {
return fmt.Errorf("failed to update all indexes, last error was: %w", lastErr)
err := fmt.Errorf("failed to update all indexes, last error was: %w", lastErr)
reg.state.ReportUpdateCheck(nil, err)
return err
}
// Get pending resources and update status.
pendingResourceVersions, _ := reg.GetPendingDownloads(true, false)
reg.state.ReportUpdateCheck(
identifiersFromResourceVersions(pendingResourceVersions),
nil,
)
return nil
}
@ -127,7 +141,7 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
}
// add resources to registry
err = reg.AddResources(cleanedData, false, true, idx.PreRelease)
err = reg.AddResources(cleanedData, idx, false, true, idx.PreRelease)
if err != nil {
log.Warningf("%s: failed to add resources: %s", reg.Name, err)
}
@ -167,37 +181,17 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
}
// DownloadUpdates checks if updates are available and downloads updates of used components.
func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
// create list of downloads
var toUpdate []*ResourceVersion
var missingSigs []*ResourceVersion
reg.RLock()
for _, res := range reg.resources {
res.Lock()
func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context, automaticOnly bool) error {
// Start registry operation.
reg.state.StartOperation(StateDownloading)
defer reg.state.EndOperation()
// check if we want to download
if res.inUse() ||
res.available() || // resource was used in the past
utils.StringInSlice(reg.MandatoryUpdates, res.Identifier) { // resource is mandatory
// add all non-available and eligible versions to update queue
for _, rv := range res.Versions {
switch {
case !rv.CurrentRelease:
// We are not interested in older releases.
case !rv.Available:
// File is not available.
toUpdate = append(toUpdate, rv)
case !rv.SigAvailable && res.VerificationOptions != nil:
// File signature is not available and verification is enabled.
missingSigs = append(missingSigs, rv)
}
}
}
res.Unlock()
}
reg.RUnlock()
// Get pending updates.
toUpdate, missingSigs := reg.GetPendingDownloads(!automaticOnly, true)
downloadDetailsResources := identifiersFromResourceVersions(toUpdate)
reg.state.UpdateOperationDetails(&StateDownloadingDetails{
Resources: downloadDetailsResources,
})
// nothing to update
if len(toUpdate) == 0 && len(missingSigs) == 0 {
@ -211,63 +205,136 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
}
// download updates
log.Infof("%s: starting to download %d updates in parallel", reg.Name, len(toUpdate)+len(missingSigs))
var wg sync.WaitGroup
wg.Add(len(toUpdate) + len(missingSigs))
log.Infof("%s: starting to download %d updates", reg.Name, len(toUpdate)+len(missingSigs))
client := &http.Client{}
var reportError error
for idx := range toUpdate {
go func(rv *ResourceVersion) {
var err error
defer wg.Done()
defer func() {
if x := recover(); x != nil {
log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x)
}
}()
for tries := 0; tries < 3; tries++ {
err = reg.fetchFile(ctx, client, rv, tries)
if err == nil {
rv.Available = true
return
}
}
if err != nil {
log.Warningf("%s: failed to download %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
}
}(toUpdate[idx])
}
for idx := range missingSigs {
go func(rv *ResourceVersion) {
var err error
defer wg.Done()
defer func() {
if x := recover(); x != nil {
log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x)
}
}()
for tries := 0; tries < 3; tries++ {
err = reg.fetchMissingSig(ctx, client, rv, tries)
if err == nil {
for i, rv := range toUpdate {
var err error
for tries := 0; tries < 3; tries++ {
err = reg.fetchFile(ctx, client, rv, tries)
if err == nil {
// Update resource version state.
rv.resource.Lock()
rv.Available = true
if rv.resource.VerificationOptions != nil {
rv.SigAvailable = true
return
}
rv.resource.Unlock()
break
}
if err != nil {
log.Warningf("%s: failed to download missing sig of %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
}
}(missingSigs[idx])
}
if err != nil {
reportError := fmt.Errorf("failed to download %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
log.Warningf("%s: %s", reg.Name, reportError)
}
reg.state.UpdateOperationDetails(&StateDownloadingDetails{
Resources: downloadDetailsResources,
FinishedUpTo: i + 1,
})
}
wg.Wait()
for _, rv := range missingSigs {
var err error
for tries := 0; tries < 3; tries++ {
err = reg.fetchMissingSig(ctx, client, rv, tries)
if err == nil {
// Update resource version state.
rv.resource.Lock()
rv.SigAvailable = true
rv.resource.Unlock()
break
}
}
if err != nil {
reportError := fmt.Errorf("failed to download missing sig of %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
log.Warningf("%s: %s", reg.Name, reportError)
}
}
reg.state.ReportDownloads(
downloadDetailsResources,
reportError,
)
log.Infof("%s: finished downloading updates", reg.Name)
return nil
}
// DownloadUpdates checks if updates are available and downloads updates of used components.
// GetPendingDownloads returns the list of pending downloads.
// If manual is set, indexes with AutoDownload=false will be checked.
// If auto is set, indexes with AutoDownload=true will be checked.
func (reg *ResourceRegistry) GetPendingDownloads(manual, auto bool) (resources, sigs []*ResourceVersion) {
reg.RLock()
defer reg.RUnlock()
// create list of downloads
var toUpdate []*ResourceVersion
var missingSigs []*ResourceVersion
for _, res := range reg.resources {
func() {
res.Lock()
defer res.Unlock()
// Skip resources without index or indexes that should not be reported
// according to parameters.
switch {
case res.Index == nil:
// Cannot download if resource is not part of an index.
return
case manual && !res.Index.AutoDownload:
// Manual update report and index is not auto-download.
case auto && res.Index.AutoDownload:
// Auto update report and index is auto-download.
default:
// Resource should not be reported.
return
}
// Skip resources we don't need.
switch {
case res.inUse():
// Update if resource is in use.
case res.available():
// Update if resource is available locally, ie. was used in the past.
case utils.StringInSlice(reg.MandatoryUpdates, res.Identifier):
// Update is set as mandatory.
default:
// Resource does not need to be updated.
return
}
// Go through all versions until we find versions that need updating.
for _, rv := range res.Versions {
switch {
case !rv.CurrentRelease:
// We are not interested in older releases.
case !rv.Available:
// File not available locally, download!
toUpdate = append(toUpdate, rv)
case !rv.SigAvailable && res.VerificationOptions != nil:
// File signature is not available and verification is enabled, download signature!
missingSigs = append(missingSigs, rv)
}
}
}()
}
return toUpdate, missingSigs
}
func identifiersFromResourceVersions(resourceVersions []*ResourceVersion) []string {
identifiers := make([]string, len(resourceVersions))
for i, rv := range resourceVersions {
identifiers[i] = rv.resource.Identifier
}
return identifiers
}