Merge pull request #37 from safing/feature/custom-index

Add support for custom index files.

updater/export.go

@@ -1,6 +1,7 @@
 package updater
 
-// Export exports the list of resources. All resources must be locked when accessed.
+// Export exports the list of resources. All resources must be
+// locked when accessed.
 func (reg *ResourceRegistry) Export() map[string]*Resource {
 	reg.RLock()
 	defer reg.RUnlock()

updater/fetch.go

@@ -25,7 +25,7 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error {
 	// create URL
 	downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath())
 	if err != nil {
-		return fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath(), err)
+		return fmt.Errorf("error build url (%s + %s): %w", reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath(), err)
 	}
 
 	// check destination dir
@@ -39,14 +39,14 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error {
 	// open file for writing
 	atomicFile, err := renameio.TempFile(reg.tmpDir.Path, rv.storagePath())
 	if err != nil {
-		return fmt.Errorf("could not create temp file for download: %s", err)
+		return fmt.Errorf("could not create temp file for download: %w", err)
 	}
 	defer atomicFile.Cleanup() //nolint:errcheck // ignore error for now, tmp dir will be cleaned later again anyway
 
 	// start file download
 	resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose
 	if err != nil {
-		return fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+		return fmt.Errorf("error fetching url (%s): %w", downloadURL, err)
 	}
 	defer resp.Body.Close()
 
@@ -57,7 +57,7 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error {
 	// download and write file
 	n, err := io.Copy(atomicFile, resp.Body)
 	if err != nil {
-		return fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+		return fmt.Errorf("failed downloading %s: %w", downloadURL, err)
 	}
 	if resp.ContentLength != n {
 		return fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength)
@@ -66,7 +66,7 @@ func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error {
 	// finalize file
 	err = atomicFile.CloseAtomicallyReplace()
 	if err != nil {
-		return fmt.Errorf("%s: failed to finalize file %s: %s", reg.Name, rv.storagePath(), err)
+		return fmt.Errorf("%s: failed to finalize file %s: %w", reg.Name, rv.storagePath(), err)
 	}
 	// set permissions
 	if !onWindows {
@@ -90,13 +90,13 @@ func (reg *ResourceRegistry) fetchData(downloadPath string, tries int) ([]byte,
 	// create URL
 	downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath)
 	if err != nil {
-		return nil, fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath, err)
+		return nil, fmt.Errorf("error build url (%s + %s): %w", reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath, err)
 	}
 
 	// start file download
 	resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose
 	if err != nil {
-		return nil, fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+		return nil, fmt.Errorf("error fetching url (%s): %w", downloadURL, err)
 	}
 	defer resp.Body.Close()
 
@@ -108,7 +108,7 @@ func (reg *ResourceRegistry) fetchData(downloadPath string, tries int) ([]byte,
 	buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
 	n, err := io.Copy(buf, resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+		return nil, fmt.Errorf("failed downloading %s: %w", downloadURL, err)
 	}
 	if resp.ContentLength != n {
 		return nil, fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength)

updater/get.go

@@ -37,7 +37,7 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
 	// check download dir
 	err := reg.tmpDir.Ensure()
 	if err != nil {
-		return nil, fmt.Errorf("could not prepare tmp directory for download: %s", err)
+		return nil, fmt.Errorf("could not prepare tmp directory for download: %w", err)
 	}
 
 	// download file

updater/indexes.go

@@ -0,0 +1,16 @@
+package updater
+
+// Index describes an index file pulled by the updater.
+type Index struct {
+	// Path is the path to the index file
+	// on the update server.
+	Path string
+
+	// Stable is set if the index file contains only stable
+	// releases.
+	Stable bool
+
+	// Beta is set if the index file contains beta
+	// releases.
+	Beta bool
+}

updater/registry.go

@@ -20,6 +20,7 @@ type ResourceRegistry struct {
 	Name       string
 	storageDir *utils.DirStructure
 	tmpDir     *utils.DirStructure
+	indexes    []Index
 
 	resources        map[string]*Resource
 	UpdateURLs       []string
@@ -30,6 +31,14 @@ type ResourceRegistry struct {
 	Online  bool
 }
 
+// AddIndex adds a new index to the resource registry.
+func (reg *ResourceRegistry) AddIndex(idx Index) {
+	reg.Lock()
+	defer reg.Unlock()
+
+	reg.indexes = append(reg.indexes, idx)
+}
+
 // Initialize initializes a raw registry struct and makes it ready for usage.
 func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
 	// check if storage dir is available

updater/storage.go

@@ -13,7 +13,11 @@ import (
 	"github.com/safing/portbase/utils"
 )
 
-// ScanStorage scans root within the storage dir and adds found resources to the registry. If an error occurred, it is logged and the last error is returned. Everything that was found despite errors is added to the registry anyway. Leave root empty to scan the full storage dir.
+// ScanStorage scans root within the storage dir and adds found
+// resources to the registry. If an error occurred, it is logged
+// and the last error is returned. Everything that was found
+// despite errors is added to the registry anyway. Leave root
+// empty to scan the full storage dir.
 func (reg *ResourceRegistry) ScanStorage(root string) error {
 	var lastError error
 
@@ -34,7 +38,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error {
 	// walk fs
 	_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
-			lastError = fmt.Errorf("%s: could not read %s: %s", reg.Name, path, err)
+			lastError = fmt.Errorf("%s: could not read %s: %w", reg.Name, path, err)
 			log.Warning(lastError.Error())
 			return nil
 		}
@@ -42,7 +46,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error {
 		// get relative path to storage
 		relativePath, err := filepath.Rel(reg.storageDir.Path, path)
 		if err != nil {
-			lastError = fmt.Errorf("%s: could not get relative path of %s: %s", reg.Name, path, err)
+			lastError = fmt.Errorf("%s: could not get relative path of %s: %w", reg.Name, path, err)
 			log.Warning(lastError.Error())
 			return nil
 		}
@@ -62,7 +66,7 @@ func (reg *ResourceRegistry) ScanStorage(root string) error {
 		// save
 		err = reg.AddResource(identifier, version, true, false, false)
 		if err != nil {
-			lastError = fmt.Errorf("%s: could not get add resource %s v%s: %s", reg.Name, identifier, version, err)
+			lastError = fmt.Errorf("%s: could not get add resource %s v%s: %w", reg.Name, identifier, version, err)
 			log.Warning(lastError.Error())
 		}
 		return nil
@@ -71,29 +75,40 @@ func (reg *ResourceRegistry) ScanStorage(root string) error {
 	return lastError
 }
 
-// LoadIndexes loads the current release indexes from disk and will fetch a new version if not available and online.
+// LoadIndexes loads the current release indexes from disk
+// or will fetch a new version if not available and the
+// registry is marked as online.
 func (reg *ResourceRegistry) LoadIndexes() error {
-	err := reg.loadIndexFile("stable.json", true, false)
-	if err != nil {
-		err = reg.downloadIndex("stable.json", true, false)
-		if err != nil {
-			return err
+	var firstErr error
+	for _, idx := range reg.getIndexes() {
+		err := reg.loadIndexFile(idx)
+		if err != nil && reg.Online {
+			// try to download the index file if a local disk version
+			// does not exist or we don't have permission to read it.
+			if os.IsNotExist(err) || os.IsPermission(err) {
+				err = reg.downloadIndex(idx)
+			}
+		}
+
+		if err != nil && firstErr == nil {
+			firstErr = err
 		}
 	}
 
-	err = reg.loadIndexFile("beta.json", false, true)
-	if err != nil {
-		err = reg.downloadIndex("beta.json", false, true)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
+	return firstErr
 }
 
-func (reg *ResourceRegistry) loadIndexFile(name string, stableRelease, betaRelease bool) error {
-	data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, name))
+func (reg *ResourceRegistry) getIndexes() []Index {
+	reg.RLock()
+	defer reg.RUnlock()
+	indexes := make([]Index, len(reg.indexes))
+	copy(indexes, reg.indexes)
+	return indexes
+}
+
+func (reg *ResourceRegistry) loadIndexFile(idx Index) error {
+	path := filepath.FromSlash(idx.Path)
+	data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, path))
 	if err != nil {
 		return err
 	}
@@ -105,26 +120,26 @@ func (reg *ResourceRegistry) loadIndexFile(name string, stableRelease, betaRelea
 	}
 
 	if len(releases) == 0 {
-		return fmt.Errorf("%s is empty", name)
+		return fmt.Errorf("%s is empty", path)
 	}
 
-	err = reg.AddResources(releases, false, stableRelease, betaRelease)
+	err = reg.AddResources(releases, false, idx.Stable, idx.Beta)
 	if err != nil {
 		log.Warningf("%s: failed to add resource: %s", reg.Name, err)
 	}
 	return nil
 }
 
-// CreateSymlinks creates a directory structure with unversions symlinks to the given updates list.
+// CreateSymlinks creates a directory structure with unversioned symlinks to the given updates list.
 func (reg *ResourceRegistry) CreateSymlinks(symlinkRoot *utils.DirStructure) error {
 	err := os.RemoveAll(symlinkRoot.Path)
 	if err != nil {
-		return fmt.Errorf("failed to wipe symlink root: %s", err)
+		return fmt.Errorf("failed to wipe symlink root: %w", err)
 	}
 
 	err = symlinkRoot.Ensure()
 	if err != nil {
-		return fmt.Errorf("failed to create symlink root: %s", err)
+		return fmt.Errorf("failed to create symlink root: %w", err)
 	}
 
 	reg.RLock()
@@ -141,17 +156,17 @@ func (reg *ResourceRegistry) CreateSymlinks(symlinkRoot *utils.DirStructure) err
 
 		err = symlinkRoot.EnsureAbsPath(linkPathDir)
 		if err != nil {
-			return fmt.Errorf("failed to create dir for link: %s", err)
+			return fmt.Errorf("failed to create dir for link: %w", err)
 		}
 
 		relativeTargetPath, err := filepath.Rel(linkPathDir, targetPath)
 		if err != nil {
-			return fmt.Errorf("failed to get relative target path: %s", err)
+			return fmt.Errorf("failed to get relative target path: %w", err)
 		}
 
 		err = os.Symlink(relativeTargetPath, linkPath)
 		if err != nil {
-			return fmt.Errorf("failed to link %s: %s", res.Identifier, err)
+			return fmt.Errorf("failed to link %s: %w", res.Identifier, err)
 		}
 	}
 

updater/updating.go

@@ -13,53 +13,58 @@ import (
 	"github.com/safing/portbase/log"
 )
 
-// UpdateIndexes downloads the current update indexes.
+// UpdateIndexes downloads all indexes and returns the first error encountered.
 func (reg *ResourceRegistry) UpdateIndexes() error {
-	err := reg.downloadIndex("stable.json", true, false)
-	if err != nil {
-		return err
+	var firstErr error
+
+	for _, idx := range reg.getIndexes() {
+		if err := reg.downloadIndex(idx); err != nil {
+			if firstErr == nil {
+				firstErr = err
+			}
+		}
 	}
 
-	return reg.downloadIndex("beta.json", false, true)
+	return firstErr
 }
 
-func (reg *ResourceRegistry) downloadIndex(name string, stableRelease, betaRelease bool) error {
+func (reg *ResourceRegistry) downloadIndex(idx Index) error {
 	var err error
 	var data []byte
 
 	// download new index
 	for tries := 0; tries < 3; tries++ {
-		data, err = reg.fetchData(name, tries)
+		data, err = reg.fetchData(idx.Path, tries)
 		if err == nil {
 			break
 		}
 	}
 	if err != nil {
-		return fmt.Errorf("failed to download index %s: %s", name, err)
+		return fmt.Errorf("failed to download index %s: %w", idx.Path, err)
 	}
 
 	// parse
 	new := make(map[string]string)
 	err = json.Unmarshal(data, &new)
 	if err != nil {
-		return fmt.Errorf("failed to parse index %s: %s", name, err)
+		return fmt.Errorf("failed to parse index %s: %w", idx.Path, err)
 	}
 
 	// check for content
 	if len(new) == 0 {
-		return fmt.Errorf("index %s is empty", name)
+		return fmt.Errorf("index %s is empty", idx.Path)
 	}
 
 	// add resources to registry
-	_ = reg.AddResources(new, false, stableRelease, betaRelease)
+	_ = reg.AddResources(new, false, idx.Stable, idx.Beta)
 
 	// save index
-	err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, name), data, 0644)
+	err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, idx.Path), data, 0644)
 	if err != nil {
-		log.Warningf("%s: failed to save updated index %s: %s", reg.Name, name, err)
+		log.Warningf("%s: failed to save updated index %s: %s", reg.Name, idx.Path, err)
 	}
 
-	log.Infof("%s: updated index %s", reg.Name, name)
+	log.Infof("%s: updated index %s", reg.Name, idx.Path)
 	return nil
 }
 
@@ -97,7 +102,7 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
 	// check download dir
 	err := reg.tmpDir.Ensure()
 	if err != nil {
-		return fmt.Errorf("could not prepare tmp directory for download: %s", err)
+		return fmt.Errorf("could not prepare tmp directory for download: %w", err)
 	}
 
 	// download updates