Merge pull request #18 from safing/feature/move-updater

Move portmaster/updates to portbase/updater, transform to lib

.golangci.yml

@@ -4,3 +4,6 @@ linters:
     - lll
     - gochecknoinits
     - gochecknoglobals
+    - funlen
+    - whitespace
+    - wsl

Gopkg.lock

@@ -123,6 +123,14 @@
   revision = "ac23dc3fea5d1a983c43f6a0f6e2c13f0195d8bd"
   version = "v1.2.0"
 
+[[projects]]
+  digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
+  name = "github.com/inconshreveable/mousetrap"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
+  version = "v1.0"
+
 [[projects]]
   branch = "master"
   digest = "1:7e8b852581596acce37bcb939a05d7d5ff27156045b50057e659e299c16fc1ca"
@@ -208,6 +216,22 @@
   pruneopts = "UT"
   revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
 
+[[projects]]
+  digest = "1:e096613fb7cf34743d49af87d197663cfccd61876e2219853005a57baedfa562"
+  name = "github.com/spf13/cobra"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "f2b07da1e2c38d5f12845a4f607e2e1018cbb1f5"
+  version = "v0.0.5"
+
+[[projects]]
+  digest = "1:524b71991fc7d9246cc7dc2d9e0886ccb97648091c63e30eef619e6862c955dd"
+  name = "github.com/spf13/pflag"
+  packages = ["."]
+  pruneopts = "UT"
+  revision = "2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab"
+  version = "v1.0.5"
+
 [[projects]]
   branch = "master"
   digest = "1:93d6687fc19da8a35c7352d72117a6acd2072dfb7e9bfd65646227bf2a913b2a"
@@ -312,6 +336,7 @@
     "github.com/satori/go.uuid",
     "github.com/seehuhn/fortuna",
     "github.com/shirou/gopsutil/host",
+    "github.com/spf13/cobra",
     "github.com/tevino/abool",
     "github.com/tidwall/gjson",
     "github.com/tidwall/sjson",

updater/doc.go

@@ -0,0 +1,2 @@
+// Package updater is an update registry that manages updates and versions.
+package updater

updater/export.go

@@ -0,0 +1,15 @@
+package updater
+
+// Export exports the list of resources. All resources must be locked when accessed.
+func (reg *ResourceRegistry) Export() map[string]*Resource {
+	reg.RLock()
+	defer reg.RUnlock()
+
+	// copy the map
+	new := make(map[string]*Resource)
+	for key, val := range reg.resources {
+		new[key] = val
+	}
+
+	return new
+}

updater/fetch.go

@@ -0,0 +1,120 @@
+package updater
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/google/renameio"
+
+	"github.com/safing/portbase/log"
+)
+
+func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error {
+	// backoff when retrying
+	if tries > 0 {
+		time.Sleep(time.Duration(tries*tries) * time.Second)
+	}
+
+	// create URL
+	downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath())
+	if err != nil {
+		return fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath(), err)
+	}
+
+	// check destination dir
+	dirPath := filepath.Dir(rv.storagePath())
+
+	err = reg.storageDir.EnsureAbsPath(dirPath)
+	if err != nil {
+		return fmt.Errorf("could not create updates folder: %s", dirPath)
+	}
+
+	// open file for writing
+	atomicFile, err := renameio.TempFile(reg.tmpDir.Path, rv.storagePath())
+	if err != nil {
+		return fmt.Errorf("could not create temp file for download: %s", err)
+	}
+	defer atomicFile.Cleanup() //nolint:errcheck // ignore error for now, tmp dir will be cleaned later again anyway
+
+	// start file download
+	resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose
+	if err != nil {
+		return fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+	}
+	defer resp.Body.Close()
+
+	// download and write file
+	n, err := io.Copy(atomicFile, resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+	}
+	if resp.ContentLength != n {
+		return fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength)
+	}
+
+	// finalize file
+	err = atomicFile.CloseAtomicallyReplace()
+	if err != nil {
+		return fmt.Errorf("%s: failed to finalize file %s: %s", reg.Name, rv.storagePath(), err)
+	}
+	// set permissions
+	if !onWindows {
+		// TODO: only set executable files to 0755, set other to 0644
+		err = os.Chmod(rv.storagePath(), 0755)
+		if err != nil {
+			log.Warningf("%s: failed to set permissions on downloaded file %s: %s", reg.Name, rv.storagePath(), err)
+		}
+	}
+
+	log.Infof("%s: fetched %s (stored to %s)", reg.Name, downloadURL, rv.storagePath())
+	return nil
+}
+
+func (reg *ResourceRegistry) fetchData(downloadPath string, tries int) ([]byte, error) {
+	// backoff when retrying
+	if tries > 0 {
+		time.Sleep(time.Duration(tries*tries) * time.Second)
+	}
+
+	// create URL
+	downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath)
+	if err != nil {
+		return nil, fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath, err)
+	}
+
+	// start file download
+	resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose
+	if err != nil {
+		return nil, fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+	}
+	defer resp.Body.Close()
+
+	// download and write file
+	buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
+	n, err := io.Copy(buf, resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+	}
+	if resp.ContentLength != n {
+		return nil, fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength)
+	}
+
+	return buf.Bytes(), nil
+}
+
+func joinURLandPath(baseURL, urlPath string) (string, error) {
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return "", err
+	}
+
+	u.Path = path.Join(u.Path, urlPath)
+	return u.String(), nil
+}

updater/file.go

@@ -0,0 +1,42 @@
+package updater
+
+// File represents a file from the update system.
+type File struct {
+	resource      *Resource
+	version       *ResourceVersion
+	notifier      *notifier
+	versionedPath string
+	storagePath   string
+}
+
+// Identifier returns the identifier of the file.
+func (file *File) Identifier() string {
+	return file.resource.Identifier
+}
+
+// Version returns the version of the file.
+func (file *File) Version() string {
+	return file.version.VersionNumber
+}
+
+// Path returns the absolute filepath of the file.
+func (file *File) Path() string {
+	return file.storagePath
+}
+
+// Blacklist notifies the update system that this file is somehow broken, and should be ignored from now on, until restarted.
+func (file *File) Blacklist() error {
+	return file.resource.Blacklist(file.version.VersionNumber)
+}
+
+// used marks the file as active
+func (file *File) markActiveWithLocking() {
+	file.resource.Lock()
+	defer file.resource.Unlock()
+
+	// update last used version
+	if file.resource.ActiveVersion != file.version {
+		file.resource.ActiveVersion = file.version
+		file.resource.registry.notifyOfChanges()
+	}
+}

updater/filename.go

@@ -0,0 +1,46 @@
+package updater
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var (
+	fileVersionRegex = regexp.MustCompile(`_v([0-9]+-[0-9]+-[0-9]+b?|0)`)
+	rawVersionRegex  = regexp.MustCompile(`^([0-9]+\.[0-9]+\.[0-9]+b?\*?|0)$`)
+)
+
+// GetIdentifierAndVersion splits the given file path into its identifier and version.
+func GetIdentifierAndVersion(versionedPath string) (identifier, version string, ok bool) {
+	// extract version
+	rawVersion := fileVersionRegex.FindString(versionedPath)
+	if rawVersion == "" {
+		return "", "", false
+	}
+
+	// replace - with . and trim _
+	version = strings.Replace(strings.TrimLeft(rawVersion, "_v"), "-", ".", -1)
+
+	// put together without version
+	i := strings.Index(versionedPath, rawVersion)
+	if i < 0 {
+		// extracted version not in string (impossible)
+		return "", "", false
+	}
+	return versionedPath[:i] + versionedPath[i+len(rawVersion):], version, true
+}
+
+// GetVersionedPath combines the identifier and version and returns it as a file path.
+func GetVersionedPath(identifier, version string) (versionedPath string) {
+	// split in half
+	splittedFilePath := strings.SplitN(identifier, ".", 2)
+	// replace . with -
+	transformedVersion := strings.Replace(version, ".", "-", -1)
+
+	// put together
+	if len(splittedFilePath) == 1 {
+		return fmt.Sprintf("%s_v%s", splittedFilePath[0], transformedVersion)
+	}
+	return fmt.Sprintf("%s_v%s.%s", splittedFilePath[0], transformedVersion, splittedFilePath[1])
+}

updater/filename_test.go

@@ -0,0 +1,53 @@
+package updater
+
+import (
+	"regexp"
+	"testing"
+)
+
+func testRegexMatch(t *testing.T, testRegex *regexp.Regexp, testString string, shouldMatch bool) {
+	if testRegex.MatchString(testString) != shouldMatch {
+		if shouldMatch {
+			t.Errorf("regex %s should match %s", testRegex, testString)
+		} else {
+			t.Errorf("regex %s should not match %s", testRegex, testString)
+		}
+	}
+}
+
+func testRegexFind(t *testing.T, testRegex *regexp.Regexp, testString string, shouldMatch bool) {
+	if (testRegex.FindString(testString) != "") != shouldMatch {
+		if shouldMatch {
+			t.Errorf("regex %s should find %s", testRegex, testString)
+		} else {
+			t.Errorf("regex %s should not find %s", testRegex, testString)
+		}
+	}
+}
+
+func TestRegexes(t *testing.T) {
+	testRegexMatch(t, rawVersionRegex, "0", true)
+	testRegexMatch(t, rawVersionRegex, "0.1.2", true)
+	testRegexMatch(t, rawVersionRegex, "0.1.2*", true)
+	testRegexMatch(t, rawVersionRegex, "0.1.2b", true)
+	testRegexMatch(t, rawVersionRegex, "0.1.2b*", true)
+	testRegexMatch(t, rawVersionRegex, "12.13.14", true)
+
+	testRegexMatch(t, rawVersionRegex, "v0.1.2", false)
+	testRegexMatch(t, rawVersionRegex, "0.", false)
+	testRegexMatch(t, rawVersionRegex, "0.1", false)
+	testRegexMatch(t, rawVersionRegex, "0.1.", false)
+	testRegexMatch(t, rawVersionRegex, ".1.2", false)
+	testRegexMatch(t, rawVersionRegex, ".1.", false)
+	testRegexMatch(t, rawVersionRegex, "012345", false)
+
+	testRegexFind(t, fileVersionRegex, "/path/to/file_v0", true)
+	testRegexFind(t, fileVersionRegex, "/path/to/file_v1-2-3", true)
+	testRegexFind(t, fileVersionRegex, "/path/to/file_v1-2-3.exe", true)
+
+	testRegexFind(t, fileVersionRegex, "/path/to/file-v1-2-3", false)
+	testRegexFind(t, fileVersionRegex, "/path/to/file_v1.2.3", false)
+	testRegexFind(t, fileVersionRegex, "/path/to/file_1-2-3", false)
+	testRegexFind(t, fileVersionRegex, "/path/to/file_v1-2", false)
+	testRegexFind(t, fileVersionRegex, "/path/to/file-v1-2-3", false)
+}

updater/get.go

@@ -0,0 +1,56 @@
+package updater
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/safing/portbase/log"
+)
+
+// Errors
+var (
+	ErrNotFound            = errors.New("the requested file could not be found")
+	ErrNotAvailableLocally = errors.New("the requested file is not available locally")
+)
+
+// GetFile returns the selected (mostly newest) file with the given identifier or an error, if it fails.
+func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
+	reg.RLock()
+	res, ok := reg.resources[identifier]
+	reg.RUnlock()
+	if !ok {
+		return nil, ErrNotFound
+	}
+
+	file := res.GetFile()
+	// check if file is available locally
+	if file.version.Available {
+		file.markActiveWithLocking()
+		return file, nil
+	}
+
+	// check if online
+	if !reg.Online {
+		return nil, ErrNotAvailableLocally
+	}
+
+	// check download dir
+	err := reg.tmpDir.Ensure()
+	if err != nil {
+		return nil, fmt.Errorf("could not prepare tmp directory for download: %s", err)
+	}
+
+	// download file
+	log.Tracef("%s: starting download of %s", reg.Name, file.versionedPath)
+	for tries := 0; tries < 5; tries++ {
+		err = reg.fetchFile(file.version, tries)
+		if err != nil {
+			log.Tracef("%s: failed to download %s: %s, retrying (%d)", reg.Name, file.versionedPath, err, tries+1)
+		} else {
+			file.markActiveWithLocking()
+			return file, nil
+		}
+	}
+	log.Warningf("%s: failed to download %s: %s", reg.Name, file.versionedPath, err)
+	return nil, err
+}

updater/notifier.go

@@ -0,0 +1,56 @@
+package updater
+
+import (
+	"github.com/tevino/abool"
+)
+
+type notifier struct {
+	upgradeAvailable *abool.AtomicBool
+	notifyChannel    chan struct{}
+}
+
+func newNotifier() *notifier {
+	return &notifier{
+		upgradeAvailable: abool.NewBool(false),
+		notifyChannel:    make(chan struct{}),
+	}
+}
+
+func (n *notifier) markAsUpgradeable() {
+	if n.upgradeAvailable.SetToIf(false, true) {
+		close(n.notifyChannel)
+	}
+}
+
+// UpgradeAvailable returns whether an upgrade is available for this file.
+func (file *File) UpgradeAvailable() bool {
+	return file.notifier.upgradeAvailable.IsSet()
+}
+
+// WaitForAvailableUpgrade blocks (selectable) until an upgrade for this file is available.
+func (file *File) WaitForAvailableUpgrade() <-chan struct{} {
+	return file.notifier.notifyChannel
+}
+
+// registry wide change notifications
+
+func (reg *ResourceRegistry) notifyOfChanges() {
+	if !reg.notifyHooksEnabled.IsSet() {
+		return
+	}
+
+	reg.RLock()
+	defer reg.RUnlock()
+
+	for _, hook := range reg.notifyHooks {
+		go hook()
+	}
+}
+
+// RegisterNotifyHook registers a function that is called (as a goroutine) every time the resource registry changes.
+func (reg *ResourceRegistry) RegisterNotifyHook(fn func()) {
+	reg.Lock()
+	defer reg.Unlock()
+
+	reg.notifyHooks = append(reg.notifyHooks, fn)
+}

updater/registry.go

@@ -0,0 +1,168 @@
+package updater
+
+import (
+	"os"
+	"runtime"
+	"sync"
+
+	"github.com/tevino/abool"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/utils"
+)
+
+const (
+	onWindows = runtime.GOOS == "windows"
+)
+
+// ResourceRegistry is a registry for managing update resources.
+type ResourceRegistry struct {
+	sync.RWMutex
+
+	Name       string
+	storageDir *utils.DirStructure
+	tmpDir     *utils.DirStructure
+
+	resources        map[string]*Resource
+	UpdateURLs       []string
+	MandatoryUpdates []string
+
+	Beta    bool
+	DevMode bool
+	Online  bool
+
+	notifyHooks        []func()
+	notifyHooksEnabled *abool.AtomicBool
+}
+
+// Initialize initializes a raw registry struct and makes it ready for usage.
+func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
+	// check if storage dir is available
+	err := storageDir.Ensure()
+	if err != nil {
+		return err
+	}
+
+	// set default name
+	if reg.Name == "" {
+		reg.Name = "updater"
+	}
+
+	// initialize private attributes
+	reg.storageDir = storageDir
+	reg.tmpDir = storageDir.ChildDir("tmp", 0700)
+	reg.resources = make(map[string]*Resource)
+	reg.notifyHooksEnabled = abool.NewBool(true)
+
+	return nil
+}
+
+// StorageDir returns the main storage dir of the resource registry.
+func (reg *ResourceRegistry) StorageDir() *utils.DirStructure {
+	return reg.storageDir
+}
+
+// TmpDir returns the temporary working dir of the resource registry.
+func (reg *ResourceRegistry) TmpDir() *utils.DirStructure {
+	return reg.tmpDir
+}
+
+// SetDevMode sets the development mode flag.
+func (reg *ResourceRegistry) SetDevMode(on bool) {
+	reg.Lock()
+	defer reg.Unlock()
+
+	reg.DevMode = on
+}
+
+// SetBeta sets the beta flag.
+func (reg *ResourceRegistry) SetBeta(on bool) {
+	reg.Lock()
+	defer reg.Unlock()
+
+	reg.Beta = on
+}
+
+// AddResource adds a resource to the registry. Does _not_ select new version.
+func (reg *ResourceRegistry) AddResource(identifier, version string, available, stableRelease, betaRelease bool) error {
+	reg.Lock()
+	defer reg.Unlock()
+
+	err := reg.addResource(identifier, version, available, stableRelease, betaRelease)
+	return err
+}
+
+func (reg *ResourceRegistry) addResource(identifier, version string, available, stableRelease, betaRelease bool) error {
+	res, ok := reg.resources[identifier]
+	if !ok {
+		res = reg.newResource(identifier)
+		reg.resources[identifier] = res
+	}
+	return res.AddVersion(version, available, stableRelease, betaRelease)
+}
+
+// AddResources adds resources to the registry. Errors are logged, the last one is returned. Despite errors, non-failing resources are still added. Does _not_ select new versions.
+func (reg *ResourceRegistry) AddResources(versions map[string]string, available, stableRelease, betaRelease bool) error {
+	reg.Lock()
+	defer reg.Unlock()
+
+	// add versions and their flags to registry
+	var lastError error
+	for identifier, version := range versions {
+		lastError = reg.addResource(identifier, version, available, stableRelease, betaRelease)
+		if lastError != nil {
+			log.Warningf("%s: failed to add resource %s: %s", reg.Name, identifier, lastError)
+		}
+	}
+
+	return lastError
+}
+
+// SelectVersions selects new resource versions depending on the current registry state.
+func (reg *ResourceRegistry) SelectVersions() {
+	// only notify of changes after we are finished
+	reg.notifyHooksEnabled.UnSet()
+	defer func() {
+		reg.notifyHooksEnabled.Set()
+		reg.notifyOfChanges()
+	}()
+
+	reg.RLock()
+	defer reg.RUnlock()
+
+	for _, res := range reg.resources {
+		res.Lock()
+		res.selectVersion()
+		res.Unlock()
+	}
+}
+
+// GetSelectedVersions returns a list of the currently selected versions.
+func (reg *ResourceRegistry) GetSelectedVersions() (versions map[string]string) {
+	reg.RLock()
+	defer reg.RUnlock()
+
+	for _, res := range reg.resources {
+		res.Lock()
+		versions[res.Identifier] = res.SelectedVersion.VersionNumber
+		res.Unlock()
+	}
+
+	return
+}
+
+// Purge deletes old updates, retaining a certain amount, specified by the keep parameter. Will at least keep 2 updates per resource.
+func (reg *ResourceRegistry) Purge(keep int) {
+	reg.RLock()
+	defer reg.RUnlock()
+
+	for _, res := range reg.resources {
+		res.Purge(keep)
+	}
+}
+
+// Cleanup removes temporary files.
+func (reg *ResourceRegistry) Cleanup() error {
+	// delete download tmp dir
+	return os.RemoveAll(reg.tmpDir.Path)
+}

updater/registry_test.go

@@ -0,0 +1,38 @@
+package updater
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"github.com/safing/portbase/utils"
+)
+
+var (
+	registry *ResourceRegistry
+)
+
+func TestMain(m *testing.M) {
+	// setup
+	tmpDir, err := ioutil.TempDir("", "ci-portmaster-")
+	if err != nil {
+		panic(err)
+	}
+	registry = &ResourceRegistry{
+		Beta:    true,
+		DevMode: true,
+		Online:  true,
+	}
+	err = registry.Initialize(utils.NewDirStructure(tmpDir, 0777))
+	if err != nil {
+		panic(err)
+	}
+
+	// run
+	// call flag.Parse() here if TestMain uses flags
+	ret := m.Run()
+
+	// teardown
+	os.RemoveAll(tmpDir)
+	os.Exit(ret)
+}

updater/resource.go

@@ -0,0 +1,332 @@
+package updater
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/safing/portbase/log"
+
+	semver "github.com/hashicorp/go-version"
+)
+
+// Resource represents a resource (via an identifier) and multiple file versions.
+type Resource struct {
+	sync.Mutex
+	registry *ResourceRegistry
+	notifier *notifier
+
+	Identifier string
+	Versions   []*ResourceVersion
+
+	ActiveVersion   *ResourceVersion
+	SelectedVersion *ResourceVersion
+	ForceDownload   bool
+}
+
+// ResourceVersion represents a single version of a resource.
+type ResourceVersion struct {
+	resource *Resource
+
+	VersionNumber string
+	semVer        *semver.Version
+	Available     bool
+	StableRelease bool
+	BetaRelease   bool
+	Blacklisted   bool
+}
+
+// Len is the number of elements in the collection. (sort.Interface for Versions)
+func (res *Resource) Len() int {
+	return len(res.Versions)
+}
+
+// Less reports whether the element with index i should sort before the element with index j. (sort.Interface for Versions)
+func (res *Resource) Less(i, j int) bool {
+	return res.Versions[i].semVer.GreaterThan(res.Versions[j].semVer)
+}
+
+// Swap swaps the elements with indexes i and j. (sort.Interface for Versions)
+func (res *Resource) Swap(i, j int) {
+	res.Versions[i], res.Versions[j] = res.Versions[j], res.Versions[i]
+}
+
+// available returns whether any version of the resource is available.
+func (res *Resource) available() bool {
+	for _, rv := range res.Versions {
+		if rv.Available {
+			return true
+		}
+	}
+	return false
+}
+
+func (reg *ResourceRegistry) newResource(identifier string) *Resource {
+	return &Resource{
+		registry:   reg,
+		Identifier: identifier,
+		Versions:   make([]*ResourceVersion, 0, 1),
+	}
+}
+
+// AddVersion adds a resource version to a resource.
+func (res *Resource) AddVersion(version string, available, stableRelease, betaRelease bool) error {
+	res.Lock()
+	defer res.Unlock()
+
+	// reset stable or beta release flags
+	if stableRelease || betaRelease {
+		for _, rv := range res.Versions {
+			if stableRelease {
+				rv.StableRelease = false
+			}
+			if betaRelease {
+				rv.BetaRelease = false
+			}
+		}
+	}
+
+	var rv *ResourceVersion
+	// check for existing version
+	for _, possibleMatch := range res.Versions {
+		if possibleMatch.VersionNumber == version {
+			rv = possibleMatch
+			break
+		}
+	}
+
+	// create new version if none found
+	if rv == nil {
+		// parse to semver
+		sv, err := semver.NewVersion(version)
+		if err != nil {
+			return err
+		}
+
+		rv = &ResourceVersion{
+			resource:      res,
+			VersionNumber: version,
+			semVer:        sv,
+		}
+		res.Versions = append(res.Versions, rv)
+	}
+
+	// set flags
+	if available {
+		rv.Available = true
+	}
+	if stableRelease {
+		rv.StableRelease = true
+	}
+	if betaRelease {
+		rv.BetaRelease = true
+	}
+
+	return nil
+}
+
+// GetFile returns the selected version as a *File.
+func (res *Resource) GetFile() *File {
+	res.Lock()
+	defer res.Unlock()
+
+	// check for notifier
+	if res.notifier == nil {
+		// create new notifier
+		res.notifier = newNotifier()
+	}
+
+	// check if version is selected
+	if res.SelectedVersion == nil {
+		res.selectVersion()
+	}
+
+	// create file
+	return &File{
+		resource:      res,
+		version:       res.SelectedVersion,
+		notifier:      res.notifier,
+		versionedPath: res.SelectedVersion.versionedPath(),
+		storagePath:   res.SelectedVersion.storagePath(),
+	}
+}
+
+//nolint:gocognit // function already kept as simlpe as possible
+func (res *Resource) selectVersion() {
+	sort.Sort(res)
+
+	// export after we finish
+	defer func() {
+		if res.ActiveVersion != nil && // resource has already been used
+			res.SelectedVersion != res.ActiveVersion && // new selected version does not match previously selected version
+			res.notifier != nil {
+			res.notifier.markAsUpgradeable()
+			res.notifier = nil
+		}
+
+		res.registry.notifyOfChanges()
+	}()
+
+	if len(res.Versions) == 0 {
+		// TODO: find better way to deal with an empty version slice (which should not happen)
+		res.SelectedVersion = nil
+		return
+	}
+
+	// Target selection
+	// 1) Dev release if dev mode is active and ignore blacklisting
+	if res.registry.DevMode {
+		// get last element
+		rv := res.Versions[len(res.Versions)-1]
+		// check if it's a dev version
+		if rv.VersionNumber == "0" && rv.Available {
+			res.SelectedVersion = rv
+			return
+		}
+	}
+
+	// 2) Beta release if beta is active
+	if res.registry.Beta {
+		for _, rv := range res.Versions {
+			if rv.BetaRelease {
+				if !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
+					res.SelectedVersion = rv
+					return
+				}
+				break
+			}
+		}
+	}
+
+	// 3) Stable release
+	for _, rv := range res.Versions {
+		if rv.StableRelease {
+			if !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
+				res.SelectedVersion = rv
+				return
+			}
+			break
+		}
+	}
+
+	// 4) Latest stable release
+	for _, rv := range res.Versions {
+		if !strings.HasSuffix(rv.VersionNumber, "b") && !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
+			res.SelectedVersion = rv
+			return
+		}
+	}
+
+	// 5) Latest of any type
+	for _, rv := range res.Versions {
+		if !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
+			res.SelectedVersion = rv
+			return
+		}
+	}
+
+	// 6) Default to newest
+	res.SelectedVersion = res.Versions[0]
+}
+
+// Blacklist blacklists the specified version and selects a new version.
+func (res *Resource) Blacklist(version string) error {
+	res.Lock()
+	defer res.Unlock()
+
+	// count already blacklisted entries
+	valid := 0
+	for _, rv := range res.Versions {
+		if rv.VersionNumber == "0" {
+			continue // ignore dev versions
+		}
+		if !rv.Blacklisted {
+			valid++
+		}
+	}
+	if valid <= 1 {
+		return errors.New("cannot blacklist last version") // last one, cannot blacklist!
+	}
+
+	// find version and blacklist
+	for _, rv := range res.Versions {
+		if rv.VersionNumber == version {
+			// blacklist and update
+			rv.Blacklisted = true
+			res.selectVersion()
+			return nil
+		}
+	}
+
+	return errors.New("could not find version")
+}
+
+// Purge deletes old updates, retaining a certain amount, specified by the keep parameter. Will at least keep 2 updates per resource. After purging, new versions will be selected.
+func (res *Resource) Purge(keep int) {
+	res.Lock()
+	defer res.Unlock()
+
+	// safeguard
+	if keep < 2 {
+		keep = 2
+	}
+
+	// keep versions
+	var validVersions int
+	var skippedActiveVersion bool
+	var skippedSelectedVersion bool
+	var purgeFrom int
+	for i, rv := range res.Versions {
+		// continue to purging?
+		if validVersions >= keep && // skip at least <keep> versions
+			skippedActiveVersion && // skip until active version
+			skippedSelectedVersion { // skip until selected version
+			purgeFrom = i
+			break
+		}
+
+		// keep active version
+		if !skippedActiveVersion && rv == res.ActiveVersion {
+			skippedActiveVersion = true
+		}
+
+		// keep selected version
+		if !skippedSelectedVersion && rv == res.SelectedVersion {
+			skippedSelectedVersion = true
+		}
+
+		// count valid (not blacklisted) versions
+		if !rv.Blacklisted {
+			validVersions++
+		}
+	}
+
+	// check if there is anything to purge
+	if purgeFrom < keep || purgeFrom > len(res.Versions) {
+		return
+	}
+
+	// purge phase
+	for _, rv := range res.Versions[purgeFrom:] {
+		// delete
+		err := os.Remove(rv.storagePath())
+		if err != nil {
+			log.Warningf("%s: failed to purge old resource %s: %s", res.registry.Name, rv.storagePath(), err)
+		}
+	}
+	// remove entries of deleted files
+	res.Versions = res.Versions[purgeFrom:]
+
+	res.selectVersion()
+}
+
+func (rv *ResourceVersion) versionedPath() string {
+	return GetVersionedPath(rv.resource.Identifier, rv.VersionNumber)
+}
+
+func (rv *ResourceVersion) storagePath() string {
+	return filepath.Join(rv.resource.registry.storageDir.Path, filepath.FromSlash(rv.versionedPath()))
+}

updater/resource_test.go

@@ -0,0 +1,81 @@
+package updater
+
+import (
+	"testing"
+)
+
+func TestVersionSelection(t *testing.T) {
+	res := registry.newResource("test/a")
+
+	err := res.AddVersion("1.2.3", true, true, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = res.AddVersion("1.2.4b", true, false, true)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = res.AddVersion("1.2.2", true, false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = res.AddVersion("1.2.5", false, true, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = res.AddVersion("0", true, false, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	registry.Online = true
+	registry.Beta = true
+	registry.DevMode = true
+	res.selectVersion()
+	if res.SelectedVersion.VersionNumber != "0" {
+		t.Errorf("selected version should be 0, not %s", res.SelectedVersion.VersionNumber)
+	}
+
+	registry.DevMode = false
+	res.selectVersion()
+	if res.SelectedVersion.VersionNumber != "1.2.4b" {
+		t.Errorf("selected version should be 1.2.4b, not %s", res.SelectedVersion.VersionNumber)
+	}
+
+	registry.Beta = false
+	res.selectVersion()
+	if res.SelectedVersion.VersionNumber != "1.2.5" {
+		t.Errorf("selected version should be 1.2.5, not %s", res.SelectedVersion.VersionNumber)
+	}
+
+	registry.Online = false
+	res.selectVersion()
+	if res.SelectedVersion.VersionNumber != "1.2.3" {
+		t.Errorf("selected version should be 1.2.3, not %s", res.SelectedVersion.VersionNumber)
+	}
+
+	f123 := res.GetFile()
+	f123.markActiveWithLocking()
+
+	err = res.Blacklist("1.2.3")
+	if err != nil {
+		t.Fatal(err)
+	}
+	if res.SelectedVersion.VersionNumber != "1.2.2" {
+		t.Errorf("selected version should be 1.2.2, not %s", res.SelectedVersion.VersionNumber)
+	}
+
+	if !f123.UpgradeAvailable() {
+		t.Error("upgrade should be available (flag)")
+	}
+	select {
+	case <-f123.WaitForAvailableUpgrade():
+	default:
+		t.Error("upgrade should be available (chan)")
+	}
+
+	t.Logf("resource: %+v", res)
+	for _, rv := range res.Versions {
+		t.Logf("version %s: %+v", rv.VersionNumber, rv)
+	}
+}

updater/storage.go

@@ -0,0 +1,159 @@
+package updater
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/utils"
+)
+
+// ScanStorage scans root within the storage dir and adds found resources to the registry. If an error occurred, it is logged and the last error is returned. Everything that was found despite errors is added to the registry anyway. Leave root empty to scan the full storage dir.
+func (reg *ResourceRegistry) ScanStorage(root string) error {
+	var lastError error
+
+	// prep root
+	if root == "" {
+		root = reg.storageDir.Path
+	} else {
+		var err error
+		root, err = filepath.Abs(root)
+		if err != nil {
+			return err
+		}
+		if !strings.HasPrefix(root, reg.storageDir.Path) {
+			return errors.New("supplied scan root path not within storage")
+		}
+	}
+
+	// walk fs
+	_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			lastError = fmt.Errorf("%s: could not read %s: %s", reg.Name, path, err)
+			log.Warning(lastError.Error())
+			return nil
+		}
+
+		// get relative path to storage
+		relativePath, err := filepath.Rel(reg.storageDir.Path, path)
+		if err != nil {
+			lastError = fmt.Errorf("%s: could not get relative path of %s: %s", reg.Name, path, err)
+			log.Warning(lastError.Error())
+			return nil
+		}
+		// ignore files in tmp dir
+		if strings.HasPrefix(relativePath, reg.tmpDir.Path) {
+			return nil
+		}
+
+		// convert to identifier and version
+		relativePath = filepath.ToSlash(relativePath)
+		identifier, version, ok := GetIdentifierAndVersion(relativePath)
+		if !ok {
+			// file does not conform to format
+			return nil
+		}
+
+		// save
+		err = reg.AddResource(identifier, version, true, false, false)
+		if err != nil {
+			lastError = fmt.Errorf("%s: could not get add resource %s v%s: %s", reg.Name, identifier, version, err)
+			log.Warning(lastError.Error())
+		}
+		return nil
+	})
+
+	return lastError
+}
+
+// LoadIndexes loads the current release indexes from disk and will fetch a new version if not available and online.
+func (reg *ResourceRegistry) LoadIndexes() error {
+	err := reg.loadIndexFile("stable.json", true, false)
+	if err != nil {
+		err = reg.downloadIndex("stable.json", true, false)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = reg.loadIndexFile("beta.json", false, true)
+	if err != nil {
+		err = reg.downloadIndex("beta.json", false, true)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (reg *ResourceRegistry) loadIndexFile(name string, stableRelease, betaRelease bool) error {
+	data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, name))
+	if err != nil {
+		return err
+	}
+
+	releases := make(map[string]string)
+	err = json.Unmarshal(data, &releases)
+	if err != nil {
+		return err
+	}
+
+	if len(releases) == 0 {
+		return fmt.Errorf("%s is empty", name)
+	}
+
+	err = reg.AddResources(releases, false, stableRelease, betaRelease)
+	if err != nil {
+		log.Warningf("%s: failed to add resource: %s", reg.Name, err)
+	}
+	return nil
+}
+
+// CreateSymlinks creates a directory structure with unversions symlinks to the given updates list.
+func (reg *ResourceRegistry) CreateSymlinks(symlinkRoot *utils.DirStructure) error {
+	err := os.RemoveAll(symlinkRoot.Path)
+	if err != nil {
+		return fmt.Errorf("failed to wipe symlink root: %s", err)
+	}
+
+	err = symlinkRoot.Ensure()
+	if err != nil {
+		return fmt.Errorf("failed to create symlink root: %s", err)
+	}
+
+	reg.RLock()
+	defer reg.RUnlock()
+
+	for _, res := range reg.resources {
+		if res.SelectedVersion == nil {
+			return fmt.Errorf("no selected version available for %s", res.Identifier)
+		}
+
+		targetPath := res.SelectedVersion.storagePath()
+		linkPath := filepath.Join(symlinkRoot.Path, filepath.FromSlash(res.Identifier))
+		linkPathDir := filepath.Dir(linkPath)
+
+		err = symlinkRoot.EnsureAbsPath(linkPathDir)
+		if err != nil {
+			return fmt.Errorf("failed to create dir for link: %s", err)
+		}
+
+		relativeTargetPath, err := filepath.Rel(linkPathDir, targetPath)
+		if err != nil {
+			return fmt.Errorf("failed to get relative target path: %s", err)
+		}
+
+		err = os.Symlink(relativeTargetPath, linkPath)
+		if err != nil {
+			return fmt.Errorf("failed to link %s: %s", res.Identifier, err)
+		}
+	}
+
+	return nil
+}

updater/storage_test.go

@@ -0,0 +1,68 @@
+package updater
+
+/*
+func testLoadLatestScope(t *testing.T, basePath, filePath, expectedIdentifier, expectedVersion string) {
+	fullPath := filepath.Join(basePath, filePath)
+
+	// create dir
+	dirPath := filepath.Dir(fullPath)
+	err := os.MkdirAll(dirPath, 0755)
+	if err != nil {
+		t.Fatalf("could not create test dir: %s\n", err)
+		return
+	}
+
+	// touch file
+	err = ioutil.WriteFile(fullPath, []byte{}, 0644)
+	if err != nil {
+		t.Fatalf("could not create test file: %s\n", err)
+		return
+	}
+
+	// run loadLatestScope
+	latest, err := ScanForLatest(basePath, true)
+	if err != nil {
+		t.Errorf("could not update latest: %s\n", err)
+		return
+	}
+	for key, val := range latest {
+		localUpdates[key] = val
+	}
+
+	// test result
+	version, ok := localUpdates[expectedIdentifier]
+	if !ok {
+		t.Errorf("identifier %s not in map", expectedIdentifier)
+		t.Errorf("current map: %v", localUpdates)
+	}
+	if version != expectedVersion {
+		t.Errorf("unexpected version for %s: %s", filePath, version)
+	}
+}
+
+func TestLoadLatestScope(t *testing.T) {
+
+	updatesLock.Lock()
+	defer updatesLock.Unlock()
+
+	tmpDir, err := ioutil.TempDir("", "testing_")
+	if err != nil {
+		t.Fatalf("could not create test dir: %s\n", err)
+		return
+	}
+	defer os.RemoveAll(tmpDir)
+
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "1.2.3")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4b.zip", "all/ui/assets.zip", "1.2.4b")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-5.zip", "all/ui/assets.zip", "1.2.5")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "1.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v2-3-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "1.2.3")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v2-1-1", "os_platform/portmaster/portmaster", "2.1.1")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "2.1.1")
+
+}
+*/

updater/updating.go

@@ -0,0 +1,125 @@
+package updater
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/safing/portbase/utils"
+
+	"github.com/safing/portbase/log"
+)
+
+// UpdateIndexes downloads the current update indexes.
+func (reg *ResourceRegistry) UpdateIndexes() error {
+	err := reg.downloadIndex("stable.json", true, false)
+	if err != nil {
+		return err
+	}
+
+	return reg.downloadIndex("beta.json", false, true)
+}
+
+func (reg *ResourceRegistry) downloadIndex(name string, stableRelease, betaRelease bool) error {
+	var err error
+	var data []byte
+
+	// download new index
+	for tries := 0; tries < 3; tries++ {
+		data, err = reg.fetchData(name, tries)
+		if err == nil {
+			break
+		}
+	}
+	if err != nil {
+		return fmt.Errorf("failed to download index %s: %s", name, err)
+	}
+
+	// parse
+	new := make(map[string]string)
+	err = json.Unmarshal(data, &new)
+	if err != nil {
+		return fmt.Errorf("failed to parse index %s: %s", name, err)
+	}
+
+	// check for content
+	if len(new) == 0 {
+		return fmt.Errorf("index %s is empty", name)
+	}
+
+	// add resources to registry
+	_ = reg.AddResources(new, false, stableRelease, betaRelease)
+
+	// save index
+	err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, name), data, 0644)
+	if err != nil {
+		log.Warningf("%s: failed to save updated index %s: %s", reg.Name, name, err)
+	}
+
+	log.Infof("%s: updated index %s", reg.Name, name)
+	return nil
+}
+
+// DownloadUpdates checks if updates are available and downloads updates of used components.
+func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
+	// create list of downloads
+	var toUpdate []*ResourceVersion
+	reg.RLock()
+	for _, res := range reg.resources {
+		res.Lock()
+
+		// check if we want to download
+		if res.ActiveVersion != nil || // resource is currently being used
+			res.available() || // resource was used in the past
+			utils.StringInSlice(reg.MandatoryUpdates, res.Identifier) { // resource is mandatory
+
+			// add all non-available and eligible versions to update queue
+			for _, rv := range res.Versions {
+				if !rv.Available && (rv.StableRelease || reg.Beta && rv.BetaRelease) {
+					toUpdate = append(toUpdate, rv)
+				}
+			}
+		}
+
+		res.Unlock()
+	}
+	reg.RUnlock()
+
+	// nothing to update
+	if len(toUpdate) == 0 {
+		log.Infof("%s: everything up to date", reg.Name)
+		return nil
+	}
+
+	// check download dir
+	err := reg.tmpDir.Ensure()
+	if err != nil {
+		return fmt.Errorf("could not prepare tmp directory for download: %s", err)
+	}
+
+	// download updates
+	log.Infof("%s: starting to download %d updates", reg.Name, len(toUpdate))
+	for _, rv := range toUpdate {
+		for tries := 0; tries < 3; tries++ {
+			err = reg.fetchFile(rv, tries)
+			if err == nil {
+				break
+			}
+		}
+		if err != nil {
+			log.Warningf("%s: failed to download %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
+		}
+	}
+	log.Infof("%s: finished downloading updates", reg.Name)
+
+	// remove tmp folder after we are finished
+	err = os.RemoveAll(reg.tmpDir.Path)
+	if err != nil {
+		log.Tracef("%s: failed to remove tmp dir %s after downloading updates: %s", reg.Name, reg.tmpDir.Path, err)
+	}
+
+	return nil
+}

updater/uptool/.gitignore

@@ -0,0 +1 @@
+uptool

updater/uptool/root.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/safing/portbase/updater"
+	"github.com/safing/portbase/utils"
+	"github.com/spf13/cobra"
+)
+
+var registry *updater.ResourceRegistry
+
+var rootCmd = &cobra.Command{
+	Use:   "uptool",
+	Short: "helper tool for the update process",
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return cmd.Usage()
+	},
+	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		absPath, err := filepath.Abs(args[0])
+		if err != nil {
+			return err
+		}
+
+		registry = &updater.ResourceRegistry{}
+		return registry.Initialize(utils.NewDirStructure(absPath, 0755))
+	},
+	SilenceUsage: true,
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		os.Exit(1)
+	}
+}

updater/uptool/scan.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(scanCmd)
+}
+
+var scanCmd = &cobra.Command{
+	Use:   "scan",
+	Short: "Scan the specified directory and print the result",
+	Args:  cobra.ExactArgs(1),
+	RunE:  scan,
+}
+
+func scan(cmd *cobra.Command, args []string) error {
+	err := scanStorage()
+	if err != nil {
+		return err
+	}
+
+	// export beta
+	data, err := json.MarshalIndent(exportSelected(true), "", " ")
+	if err != nil {
+		return err
+	}
+	// print
+	fmt.Println("beta:")
+	fmt.Println(string(data))
+
+	// export stable
+	data, err = json.MarshalIndent(exportSelected(false), "", " ")
+	if err != nil {
+		return err
+	}
+	// print
+	fmt.Println("\nstable:")
+	fmt.Println(string(data))
+
+	return nil
+}
+
+func scanStorage() error {
+	files, err := ioutil.ReadDir(registry.StorageDir().Path)
+	if err != nil {
+		return err
+	}
+
+	// scan "all" and all "os_platform" dirs
+	for _, file := range files {
+		if file.IsDir() && (file.Name() == "all" || strings.Contains(file.Name(), "_")) {
+			err := registry.ScanStorage(filepath.Join(registry.StorageDir().Path, file.Name()))
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func exportSelected(beta bool) map[string]string {
+	registry.SetBeta(beta)
+	registry.SelectVersions()
+	export := registry.Export()
+
+	versions := make(map[string]string)
+	for _, rv := range export {
+		versions[rv.Identifier] = rv.SelectedVersion.VersionNumber
+	}
+	return versions
+}

updater/uptool/update.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(updateCmd)
+}
+
+var updateCmd = &cobra.Command{
+	Use:   "update",
+	Short: "Update scans the specified directory and registry the index and symlink structure",
+	Args:  cobra.ExactArgs(1),
+	RunE:  update,
+}
+
+func update(cmd *cobra.Command, args []string) error {
+	err := scanStorage()
+	if err != nil {
+		return err
+	}
+
+	// export beta
+	data, err := json.MarshalIndent(exportSelected(true), "", " ")
+	if err != nil {
+		return err
+	}
+	// print
+	fmt.Println("beta:")
+	fmt.Println(string(data))
+	// write index
+	err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "beta.json"), data, 0755)
+	if err != nil {
+		return err
+	}
+
+	// export stable
+	data, err = json.MarshalIndent(exportSelected(false), "", " ")
+	if err != nil {
+		return err
+	}
+	// print
+	fmt.Println("\nstable:")
+	fmt.Println(string(data))
+	// write index
+	err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "stable.json"), data, 0755)
+	if err != nil {
+		return err
+	}
+	// create symlinks
+	err = registry.CreateSymlinks(registry.StorageDir().ChildDir("latest", 0755))
+	if err != nil {
+		return err
+	}
+	fmt.Println("\nstable symlinks created")
+
+	return nil
+}