diff --git a/updater/export.go b/updater/export.go index af5ef1d..55b6133 100644 --- a/updater/export.go +++ b/updater/export.go @@ -1,6 +1,7 @@ package updater -// Export exports the list of resources. All resources must be locked when accessed. +// Export exports the list of resources. All resources must be +// locked when accessed. func (reg *ResourceRegistry) Export() map[string]*Resource { reg.RLock() defer reg.RUnlock() diff --git a/updater/fetch.go b/updater/fetch.go index 8b7853b..c2239a8 100644 --- a/updater/fetch.go +++ b/updater/fetch.go @@ -25,7 +25,7 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error { // create URL downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath()) if err != nil { - return fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath(), err) + return fmt.Errorf("error build url (%s + %s): %w", reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath(), err) } // check destination dir @@ -39,14 +39,14 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error { // open file for writing atomicFile, err := renameio.TempFile(reg.tmpDir.Path, rv.storagePath()) if err != nil { - return fmt.Errorf("could not create temp file for download: %s", err) + return fmt.Errorf("could not create temp file for download: %w", err) } defer atomicFile.Cleanup() //nolint:errcheck // ignore error for now, tmp dir will be cleaned later again anyway // start file download resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose if err != nil { - return fmt.Errorf("error fetching url (%s): %s", downloadURL, err) + return fmt.Errorf("error fetching url (%s): %w", downloadURL, err) } defer resp.Body.Close() @@ -57,7 +57,7 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error { // download and write file n, err := io.Copy(atomicFile, resp.Body) if err != nil { - return fmt.Errorf("failed downloading %s: %s", downloadURL, err) + return fmt.Errorf("failed downloading %s: %w", downloadURL, err) } if resp.ContentLength != n { return fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength) @@ -66,7 +66,7 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error { // finalize file err = atomicFile.CloseAtomicallyReplace() if err != nil { - return fmt.Errorf("%s: failed to finalize file %s: %s", reg.Name, rv.storagePath(), err) + return fmt.Errorf("%s: failed to finalize file %s: %w", reg.Name, rv.storagePath(), err) } // set permissions if !onWindows { @@ -90,13 +90,13 @@ func (reg *ResourceRegistry) fetchData(downloadPath string, tries int) ([]byte, // create URL downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath) if err != nil { - return nil, fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath, err) + return nil, fmt.Errorf("error build url (%s + %s): %w", reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath, err) } // start file download resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose if err != nil { - return nil, fmt.Errorf("error fetching url (%s): %s", downloadURL, err) + return nil, fmt.Errorf("error fetching url (%s): %w", downloadURL, err) } defer resp.Body.Close() @@ -108,7 +108,7 @@ func (reg *ResourceRegistry) fetchData(downloadPath string, tries int) ([]byte, buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength)) n, err := io.Copy(buf, resp.Body) if err != nil { - return nil, fmt.Errorf("failed downloading %s: %s", downloadURL, err) + return nil, fmt.Errorf("failed downloading %s: %w", downloadURL, err) } if resp.ContentLength != n { return nil, fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength) diff --git a/updater/get.go b/updater/get.go index a36df3b..2659034 100644 --- a/updater/get.go +++ b/updater/get.go @@ -37,7 +37,7 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) { // check download dir err := reg.tmpDir.Ensure() if err != nil { - return nil, fmt.Errorf("could not prepare tmp directory for download: %s", err) + return nil, fmt.Errorf("could not prepare tmp directory for download: %w", err) } // download file diff --git a/updater/indexes.go b/updater/indexes.go new file mode 100644 index 0000000..fbbd82b --- /dev/null +++ b/updater/indexes.go @@ -0,0 +1,16 @@ +package updater + +// Index describes an index file pulled by the updater. +type Index struct { + // Path is the path to the index file + // on the update server. + Path string + + // Stable is set if the index file contains only stable + // releases. + Stable bool + + // Beta is set if the index file contains beta + // releases. + Beta bool +} diff --git a/updater/registry.go b/updater/registry.go index 2d914ce..fa7fe96 100644 --- a/updater/registry.go +++ b/updater/registry.go @@ -20,6 +20,7 @@ type ResourceRegistry struct { Name string storageDir *utils.DirStructure tmpDir *utils.DirStructure + indexes []Index resources map[string]*Resource UpdateURLs []string @@ -30,6 +31,14 @@ type ResourceRegistry struct { Online bool } +// AddIndex adds a new index to the resource registry. +func (reg *ResourceRegistry) AddIndex(idx Index) { + reg.Lock() + defer reg.Unlock() + + reg.indexes = append(reg.indexes, idx) +} + // Initialize initializes a raw registry struct and makes it ready for usage. func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error { // check if storage dir is available diff --git a/updater/storage.go b/updater/storage.go index 35a6e6b..fff1017 100644 --- a/updater/storage.go +++ b/updater/storage.go @@ -13,7 +13,11 @@ import ( "github.com/safing/portbase/utils" ) -// ScanStorage scans root within the storage dir and adds found resources to the registry. If an error occurred, it is logged and the last error is returned. Everything that was found despite errors is added to the registry anyway. Leave root empty to scan the full storage dir. +// ScanStorage scans root within the storage dir and adds found +// resources to the registry. If an error occurred, it is logged +// and the last error is returned. Everything that was found +// despite errors is added to the registry anyway. Leave root +// empty to scan the full storage dir. func (reg *ResourceRegistry) ScanStorage(root string) error { var lastError error @@ -34,7 +38,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error { // walk fs _ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { if err != nil { - lastError = fmt.Errorf("%s: could not read %s: %s", reg.Name, path, err) + lastError = fmt.Errorf("%s: could not read %s: %w", reg.Name, path, err) log.Warning(lastError.Error()) return nil } @@ -42,7 +46,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error { // get relative path to storage relativePath, err := filepath.Rel(reg.storageDir.Path, path) if err != nil { - lastError = fmt.Errorf("%s: could not get relative path of %s: %s", reg.Name, path, err) + lastError = fmt.Errorf("%s: could not get relative path of %s: %w", reg.Name, path, err) log.Warning(lastError.Error()) return nil } @@ -62,7 +66,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error { // save err = reg.AddResource(identifier, version, true, false, false) if err != nil { - lastError = fmt.Errorf("%s: could not get add resource %s v%s: %s", reg.Name, identifier, version, err) + lastError = fmt.Errorf("%s: could not get add resource %s v%s: %w", reg.Name, identifier, version, err) log.Warning(lastError.Error()) } return nil @@ -71,29 +75,40 @@ func (reg *ResourceRegistry) ScanStorage(root string) error { return lastError } -// LoadIndexes loads the current release indexes from disk and will fetch a new version if not available and online. +// LoadIndexes loads the current release indexes from disk +// or will fetch a new version if not available and the +// registry is marked as online. func (reg *ResourceRegistry) LoadIndexes() error { - err := reg.loadIndexFile("stable.json", true, false) - if err != nil { - err = reg.downloadIndex("stable.json", true, false) - if err != nil { - return err + var firstErr error + for _, idx := range reg.indexes { + err := reg.loadIndexFile(idx) + if err != nil && reg.Online { + // try to download the index file if a local disk version + // does not exist or we don't have permission to read it. + if os.IsNotExist(err) || os.IsPermission(err) { + err = reg.downloadIndex(idx) + } + } + + if err != nil && firstErr == nil { + firstErr = err } } - err = reg.loadIndexFile("beta.json", false, true) - if err != nil { - err = reg.downloadIndex("beta.json", false, true) - if err != nil { - return err - } - } - - return nil + return firstErr } -func (reg *ResourceRegistry) loadIndexFile(name string, stableRelease, betaRelease bool) error { - data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, name)) +func (reg *ResourceRegistry) getIndexes() []Index { + reg.RLock() + defer reg.RUnlock() + indexes := make([]Index, len(reg.indexes)) + copy(indexes, reg.indexes) + return indexes +} + +func (reg *ResourceRegistry) loadIndexFile(idx Index) error { + path := filepath.FromSlash(idx.Path) + data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, path)) if err != nil { return err } @@ -105,26 +120,26 @@ func (reg *ResourceRegistry) loadIndexFile(name string, stableRelease, betaRelea } if len(releases) == 0 { - return fmt.Errorf("%s is empty", name) + return fmt.Errorf("%s is empty", path) } - err = reg.AddResources(releases, false, stableRelease, betaRelease) + err = reg.AddResources(releases, false, idx.Stable, idx.Beta) if err != nil { log.Warningf("%s: failed to add resource: %s", reg.Name, err) } return nil } -// CreateSymlinks creates a directory structure with unversions symlinks to the given updates list. +// CreateSymlinks creates a directory structure with unversioned symlinks to the given updates list. func (reg *ResourceRegistry) CreateSymlinks(symlinkRoot *utils.DirStructure) error { err := os.RemoveAll(symlinkRoot.Path) if err != nil { - return fmt.Errorf("failed to wipe symlink root: %s", err) + return fmt.Errorf("failed to wipe symlink root: %w", err) } err = symlinkRoot.Ensure() if err != nil { - return fmt.Errorf("failed to create symlink root: %s", err) + return fmt.Errorf("failed to create symlink root: %w", err) } reg.RLock() @@ -141,17 +156,17 @@ func (reg *ResourceRegistry) CreateSymlinks(symlinkRoot *utils.DirStructure) err err = symlinkRoot.EnsureAbsPath(linkPathDir) if err != nil { - return fmt.Errorf("failed to create dir for link: %s", err) + return fmt.Errorf("failed to create dir for link: %w", err) } relativeTargetPath, err := filepath.Rel(linkPathDir, targetPath) if err != nil { - return fmt.Errorf("failed to get relative target path: %s", err) + return fmt.Errorf("failed to get relative target path: %w", err) } err = os.Symlink(relativeTargetPath, linkPath) if err != nil { - return fmt.Errorf("failed to link %s: %s", res.Identifier, err) + return fmt.Errorf("failed to link %s: %w", res.Identifier, err) } } diff --git a/updater/updating.go b/updater/updating.go index 81c8240..173498b 100644 --- a/updater/updating.go +++ b/updater/updating.go @@ -13,53 +13,58 @@ import ( "github.com/safing/portbase/log" ) -// UpdateIndexes downloads the current update indexes. +// UpdateIndexes downloads all indexes and returns the first error encountered. func (reg *ResourceRegistry) UpdateIndexes() error { - err := reg.downloadIndex("stable.json", true, false) - if err != nil { - return err + var firstErr error + + for _, idx := range reg.getIndexes() { + if err := reg.downloadIndex(idx); err != nil { + if firstErr == nil { + firstErr = err + } + } } - return reg.downloadIndex("beta.json", false, true) + return firstErr } -func (reg *ResourceRegistry) downloadIndex(name string, stableRelease, betaRelease bool) error { +func (reg *ResourceRegistry) downloadIndex(idx Index) error { var err error var data []byte // download new index for tries := 0; tries < 3; tries++ { - data, err = reg.fetchData(name, tries) + data, err = reg.fetchData(idx.Path, tries) if err == nil { break } } if err != nil { - return fmt.Errorf("failed to download index %s: %s", name, err) + return fmt.Errorf("failed to download index %s: %w", idx.Path, err) } // parse new := make(map[string]string) err = json.Unmarshal(data, &new) if err != nil { - return fmt.Errorf("failed to parse index %s: %s", name, err) + return fmt.Errorf("failed to parse index %s: %w", idx.Path, err) } // check for content if len(new) == 0 { - return fmt.Errorf("index %s is empty", name) + return fmt.Errorf("index %s is empty", idx.Path) } // add resources to registry - _ = reg.AddResources(new, false, stableRelease, betaRelease) + _ = reg.AddResources(new, false, idx.Stable, idx.Beta) // save index - err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, name), data, 0644) + err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, idx.Path), data, 0644) if err != nil { - log.Warningf("%s: failed to save updated index %s: %s", reg.Name, name, err) + log.Warningf("%s: failed to save updated index %s: %s", reg.Name, idx.Path, err) } - log.Infof("%s: updated index %s", reg.Name, name) + log.Infof("%s: updated index %s", reg.Name, idx.Path) return nil } @@ -97,7 +102,7 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { // check download dir err := reg.tmpDir.Ensure() if err != nil { - return fmt.Errorf("could not prepare tmp directory for download: %s", err) + return fmt.Errorf("could not prepare tmp directory for download: %w", err) } // download updates