vikarti.anatra/safing-portbase
1
0
Fork 0
mirror of https://github.com/safing/portbase synced 2025-09-01 01:59:48 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Move portmaster/updates to portbase/updater, transform to lib

Browse source
This commit is contained in:
Daniel 2019-10-07 16:13:35 +02:00
parent 3be9f93001
commit f1aacc5d45
20 changed files with 1567 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
Gopkg.lock generated
View file

@ -123,6 +123,14 @@
revision = "ac23dc3fea5d1a983c43f6a0f6e2c13f0195d8bd" revision = "ac23dc3fea5d1a983c43f6a0f6e2c13f0195d8bd"
version = "v1.2.0" version = "v1.2.0"
[[projects]]
digest = "1:870d441fe217b8e689d7949fef6e43efbc787e50f200cb1e70dbca9204a1d6be"
name = "github.com/inconshreveable/mousetrap"
packages = ["."]
pruneopts = "UT"
revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75"
version = "v1.0"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:7e8b852581596acce37bcb939a05d7d5ff27156045b50057e659e299c16fc1ca" digest = "1:7e8b852581596acce37bcb939a05d7d5ff27156045b50057e659e299c16fc1ca"
@ -208,6 +216,22 @@
pruneopts = "UT" pruneopts = "UT"
revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b" revision = "bb4de0191aa41b5507caa14b0650cdbddcd9280b"
[[projects]]
digest = "1:e096613fb7cf34743d49af87d197663cfccd61876e2219853005a57baedfa562"
name = "github.com/spf13/cobra"
packages = ["."]
pruneopts = "UT"
revision = "f2b07da1e2c38d5f12845a4f607e2e1018cbb1f5"
version = "v0.0.5"
[[projects]]
digest = "1:524b71991fc7d9246cc7dc2d9e0886ccb97648091c63e30eef619e6862c955dd"
name = "github.com/spf13/pflag"
packages = ["."]
pruneopts = "UT"
revision = "2e9d26c8c37aae03e3f9d4e90b7116f5accb7cab"
version = "v1.0.5"
[[projects]] [[projects]]
branch = "master" branch = "master"
digest = "1:93d6687fc19da8a35c7352d72117a6acd2072dfb7e9bfd65646227bf2a913b2a" digest = "1:93d6687fc19da8a35c7352d72117a6acd2072dfb7e9bfd65646227bf2a913b2a"
@ -312,6 +336,7 @@
"github.com/satori/go.uuid", "github.com/satori/go.uuid",
"github.com/seehuhn/fortuna", "github.com/seehuhn/fortuna",
"github.com/shirou/gopsutil/host", "github.com/shirou/gopsutil/host",
"github.com/spf13/cobra",
"github.com/tevino/abool", "github.com/tevino/abool",
"github.com/tidwall/gjson", "github.com/tidwall/gjson",
"github.com/tidwall/sjson", "github.com/tidwall/sjson",

2
updater/doc.go Normal file
View file

@ -0,0 +1,2 @@
// Package updater is an update registry that manages updates and versions.
package updater

15
updater/export.go Normal file
View file

@ -0,0 +1,15 @@
package updater
// Export exports the list of resources. All resources must be locked when accessed.
func (reg *ResourceRegistry) Export() map[string]*Resource {
reg.RLock()
defer reg.RUnlock()
// copy the map
new := make(map[string]*Resource)
for key, val := range reg.resources {
new[key] = val
}
return new
}

119
updater/fetch.go Normal file
View file

@ -0,0 +1,119 @@
package updater
import (
"bytes"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"time"
"github.com/google/renameio"
"github.com/safing/portbase/log"
)
func (reg *ResourceRegistry) fetchFile(rv *ResourceVersion, tries int) error {
// backoff when retrying
if tries > 0 {
time.Sleep(time.Duration(tries*tries) * time.Second)
}
// create URL
downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath())
if err != nil {
return fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], rv.versionedPath(), err)
}
// check destination dir
dirPath := filepath.Dir(rv.storagePath())
err = reg.storageDir.EnsureAbsPath(dirPath)
if err != nil {
return fmt.Errorf("could not create updates folder: %s", dirPath)
}
// open file for writing
atomicFile, err := renameio.TempFile(reg.tmpDir.Path, rv.storagePath())
if err != nil {
return fmt.Errorf("could not create temp file for download: %s", err)
}
defer atomicFile.Cleanup() //nolint:errcheck // ignore error for now, tmp dir will be cleaned later again anyway
// start file download
resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose
if err != nil {
return fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
}
defer resp.Body.Close()
// download and write file
n, err := io.Copy(atomicFile, resp.Body)
if err != nil {
return fmt.Errorf("failed downloading %s: %s", downloadURL, err)
}
if resp.ContentLength != n {
return fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength)
}
// finalize file
err = atomicFile.CloseAtomicallyReplace()
if err != nil {
return fmt.Errorf("%s: failed to finalize file %s: %s", reg.Name, rv.storagePath(), err)
}
// set permissions
if !onWindows {
// FIXME: only set executable files to 0755, set other to 0644
err = os.Chmod(rv.storagePath(), 0755)
if err != nil {
log.Warningf("%s: failed to set permissions on downloaded file %s: %s", reg.Name, rv.storagePath(), err)
}
}
log.Infof("%s: fetched %s (stored to %s)", reg.Name, downloadURL, rv.storagePath())
return nil
}
func (reg *ResourceRegistry) fetchData(downloadPath string, tries int) ([]byte, error) {
// backoff when retrying
if tries > 0 {
time.Sleep(time.Duration(tries*tries) * time.Second)
}
// create URL
downloadURL, err := joinURLandPath(reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath)
if err != nil {
return nil, fmt.Errorf("error build url (%s + %s): %s", reg.UpdateURLs[tries%len(reg.UpdateURLs)], downloadPath, err)
}
// start file download
resp, err := http.Get(downloadURL) //nolint:gosec // url is variable on purpose
if err != nil {
return nil, fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
}
defer resp.Body.Close()
// download and write file
buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
n, err := io.Copy(buf, resp.Body)
if err != nil {
return nil, fmt.Errorf("failed downloading %s: %s", downloadURL, err)
}
if resp.ContentLength != n {
return nil, fmt.Errorf("download unfinished, written %d out of %d bytes", n, resp.ContentLength)
}
return buf.Bytes(), nil
}
func joinURLandPath(baseURL, urlPath string) (string, error) {
u, err := url.Parse(baseURL)
if err != nil {
return "", err
}
u.Path = path.Join(u.Path, urlPath)
return u.String(), nil
}

42
updater/file.go Normal file
View file

@ -0,0 +1,42 @@
package updater
// File represents a file from the update system.
type File struct {
resource *Resource
version *ResourceVersion
notifier *notifier
versionedPath string
storagePath string
}
// Identifier returns the identifier of the file.
func (file *File) Identifier() string {
return file.resource.Identifier
}
// Version returns the version of the file.
func (file *File) Version() string {
return file.version.VersionNumber
}
// Path returns the absolute filepath of the file.
func (file *File) Path() string {
return file.storagePath
}
// Blacklist notifies the update system that this file is somehow broken, and should be ignored from now on, until restarted.
func (file *File) Blacklist() error {
return file.resource.Blacklist(file.version.VersionNumber)
}
// used marks the file as active
func (file *File) markActiveWithLocking() {
file.resource.Lock()
defer file.resource.Unlock()
// update last used version
if file.resource.ActiveVersion != file.version {
file.resource.ActiveVersion = file.version
file.resource.registry.notifyOfChanges()
}
}

46
updater/filename.go Normal file
View file

@ -0,0 +1,46 @@
package updater
import (
"fmt"
"regexp"
"strings"
)
var (
fileVersionRegex = regexp.MustCompile(`_v([0-9]+-[0-9]+-[0-9]+b?|0)`)
rawVersionRegex = regexp.MustCompile(`^([0-9]+\.[0-9]+\.[0-9]+b?\*?|0)$`)
)
// GetIdentifierAndVersion splits the given file path into its identifier and version.
func GetIdentifierAndVersion(versionedPath string) (identifier, version string, ok bool) {
// extract version
rawVersion := fileVersionRegex.FindString(versionedPath)
if rawVersion == "" {
return "", "", false
}
// replace - with . and trim _
version = strings.Replace(strings.TrimLeft(rawVersion, "_v"), "-", ".", -1)
// put together without version
i := strings.Index(versionedPath, rawVersion)
if i < 0 {
// extracted version not in string (impossible)
return "", "", false
}
return versionedPath[:i] + versionedPath[i+len(rawVersion):], version, true
}
// GetVersionedPath combines the identifier and version and returns it as a file path.
func GetVersionedPath(identifier, version string) (versionedPath string) {
// split in half
splittedFilePath := strings.SplitN(identifier, ".", 2)
// replace . with -
transformedVersion := strings.Replace(version, ".", "-", -1)
// put together
if len(splittedFilePath) == 1 {
return fmt.Sprintf("%s_v%s", splittedFilePath[0], transformedVersion)
}
return fmt.Sprintf("%s_v%s.%s", splittedFilePath[0], transformedVersion, splittedFilePath[1])
}

53
updater/filename_test.go Normal file
View file

@ -0,0 +1,53 @@
package updater
import (
"regexp"
"testing"
)
func testRegexMatch(t *testing.T, testRegex *regexp.Regexp, testString string, shouldMatch bool) {
if testRegex.MatchString(testString) != shouldMatch {
if shouldMatch {
t.Errorf("regex %s should match %s", testRegex, testString)
} else {
t.Errorf("regex %s should not match %s", testRegex, testString)
}
}
}
func testRegexFind(t *testing.T, testRegex *regexp.Regexp, testString string, shouldMatch bool) {
if (testRegex.FindString(testString) != "") != shouldMatch {
if shouldMatch {
t.Errorf("regex %s should find %s", testRegex, testString)
} else {
t.Errorf("regex %s should not find %s", testRegex, testString)
}
}
}
func TestRegexes(t *testing.T) {
testRegexMatch(t, rawVersionRegex, "0", true)
testRegexMatch(t, rawVersionRegex, "0.1.2", true)
testRegexMatch(t, rawVersionRegex, "0.1.2*", true)
testRegexMatch(t, rawVersionRegex, "0.1.2b", true)
testRegexMatch(t, rawVersionRegex, "0.1.2b*", true)
testRegexMatch(t, rawVersionRegex, "12.13.14", true)
testRegexMatch(t, rawVersionRegex, "v0.1.2", false)
testRegexMatch(t, rawVersionRegex, "0.", false)
testRegexMatch(t, rawVersionRegex, "0.1", false)
testRegexMatch(t, rawVersionRegex, "0.1.", false)
testRegexMatch(t, rawVersionRegex, ".1.2", false)
testRegexMatch(t, rawVersionRegex, ".1.", false)
testRegexMatch(t, rawVersionRegex, "012345", false)
testRegexFind(t, fileVersionRegex, "/path/to/file_v0", true)
testRegexFind(t, fileVersionRegex, "/path/to/file_v1-2-3", true)
testRegexFind(t, fileVersionRegex, "/path/to/file_v1-2-3.exe", true)
testRegexFind(t, fileVersionRegex, "/path/to/file-v1-2-3", false)
testRegexFind(t, fileVersionRegex, "/path/to/file_v1.2.3", false)
testRegexFind(t, fileVersionRegex, "/path/to/file_1-2-3", false)
testRegexFind(t, fileVersionRegex, "/path/to/file_v1-2", false)
testRegexFind(t, fileVersionRegex, "/path/to/file-v1-2-3", false)
}

56
updater/get.go Normal file
View file

@ -0,0 +1,56 @@
package updater
import (
"errors"
"fmt"
"github.com/safing/portbase/log"
)
// Errors
var (
ErrNotFound = errors.New("the requested file could not be found")
ErrNotAvailableLocally = errors.New("the requested file is not available locally")
)
// GetFile returns the selected (mostly newest) file with the given identifier or an error, if it fails.
func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
reg.RLock()
res, ok := reg.resources[identifier]
reg.RUnlock()
if !ok {
return nil, ErrNotFound
}
file := res.GetFile()
// check if file is available locally
if file.version.Available {
file.markActiveWithLocking()
return file, nil
}
// check if online
if !reg.Online {
return nil, ErrNotAvailableLocally
}
// check download dir
err := reg.tmpDir.Ensure()
if err != nil {
return nil, fmt.Errorf("could not prepare tmp directory for download: %s", err)
}
// download file
log.Tracef("%s: starting download of %s", reg.Name, file.versionedPath)
for tries := 0; tries < 5; tries++ {
err = reg.fetchFile(file.version, tries)
if err != nil {
log.Tracef("%s: failed to download %s: %s, retrying (%d)", reg.Name, file.versionedPath, err, tries+1)
} else {
file.markActiveWithLocking()
return file, nil
}
}
log.Warningf("%s: failed to download %s: %s", reg.Name, file.versionedPath, err)
return nil, err
}

56
updater/notifier.go Normal file
View file

@ -0,0 +1,56 @@
package updater
import (
"github.com/tevino/abool"
)
type notifier struct {
upgradeAvailable *abool.AtomicBool
notifyChannel chan struct{}
}
func newNotifier() *notifier {
return &notifier{
upgradeAvailable: abool.NewBool(false),
notifyChannel: make(chan struct{}),
}
}
func (n *notifier) markAsUpgradeable() {
if n.upgradeAvailable.SetToIf(false, true) {
close(n.notifyChannel)
}
}
// UpgradeAvailable returns whether an upgrade is available for this file.
func (file *File) UpgradeAvailable() bool {
return file.notifier.upgradeAvailable.IsSet()
}
// WaitForAvailableUpgrade blocks (selectable) until an upgrade for this file is available.
func (file *File) WaitForAvailableUpgrade() <-chan struct{} {
return file.notifier.notifyChannel
}
// registry wide change notifications
func (reg *ResourceRegistry) notifyOfChanges() {
if !reg.notifyHooksEnabled.IsSet() {
return
}
reg.RLock()
defer reg.RUnlock()
for _, hook := range reg.notifyHooks {
go hook()
}
}
// RegisterNotifyHook registers a function that is called (as a goroutine) every time the resource registry changes.
func (reg *ResourceRegistry) RegisterNotifyHook(fn func()) {
reg.Lock()
defer reg.Unlock()
reg.notifyHooks = append(reg.notifyHooks, fn)
}

168
updater/registry.go Normal file
View file

@ -0,0 +1,168 @@
package updater
import (
"os"
"runtime"
"sync"
"github.com/tevino/abool"
"github.com/safing/portbase/log"
"github.com/safing/portbase/utils"
)
const (
onWindows = runtime.GOOS == "windows"
)
// ResourceRegistry is a registry for managing update resources.
type ResourceRegistry struct {
sync.RWMutex
Name string
storageDir *utils.DirStructure
tmpDir *utils.DirStructure
resources map[string]*Resource
UpdateURLs []string
MandatoryUpdates []string
Beta bool
DevMode bool
Online bool
notifyHooks []func()
notifyHooksEnabled *abool.AtomicBool
}
// Initialize initializes a raw registry struct and makes it ready for usage.
func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error {
// check if storage dir is available
err := storageDir.Ensure()
if err != nil {
return err
}
// set default name
if reg.Name == "" {
reg.Name = "updater"
}
// initialize private attributes
reg.storageDir = storageDir
reg.tmpDir = storageDir.ChildDir("tmp", 0700)
reg.resources = make(map[string]*Resource)
reg.notifyHooksEnabled = abool.NewBool(true)
return nil
}
// StorageDir returns the main storage dir of the resource registry.
func (reg *ResourceRegistry) StorageDir() *utils.DirStructure {
return reg.storageDir
}
// TmpDir returns the temporary working dir of the resource registry.
func (reg *ResourceRegistry) TmpDir() *utils.DirStructure {
return reg.tmpDir
}
// SetDevMode sets the development mode flag.
func (reg *ResourceRegistry) SetDevMode(on bool) {
reg.Lock()
defer reg.Unlock()
reg.DevMode = on
}
// SetBeta sets the beta flag.
func (reg *ResourceRegistry) SetBeta(on bool) {
reg.Lock()
defer reg.Unlock()
reg.Beta = on
}
// AddResource adds a resource to the registry. Does _not_ select new version.
func (reg *ResourceRegistry) AddResource(identifier, version string, available, stableRelease, betaRelease bool) error {
reg.Lock()
defer reg.Unlock()
err := reg.addResource(identifier, version, available, stableRelease, betaRelease)
return err
}
func (reg *ResourceRegistry) addResource(identifier, version string, available, stableRelease, betaRelease bool) error {
res, ok := reg.resources[identifier]
if !ok {
res = reg.newResource(identifier)
reg.resources[identifier] = res
}
return res.AddVersion(version, available, stableRelease, betaRelease)
}
// AddResources adds resources to the registry. Errors are logged, the last one is returned. Despite errors, non-failing resources are still added. Does _not_ select new versions.
func (reg *ResourceRegistry) AddResources(versions map[string]string, available, stableRelease, betaRelease bool) error {
reg.Lock()
defer reg.Unlock()
// add versions and their flags to registry
var lastError error
for identifier, version := range versions {
lastError = reg.addResource(identifier, version, available, stableRelease, betaRelease)
if lastError != nil {
log.Warningf("%s: failed to add resource %s: %s", reg.Name, identifier, lastError)
}
}
return lastError
}
// SelectVersions selects new resource versions depending on the current registry state.
func (reg *ResourceRegistry) SelectVersions() {
// only notify of changes after we are finished
reg.notifyHooksEnabled.UnSet()
defer func() {
reg.notifyHooksEnabled.Set()
reg.notifyOfChanges()
}()
reg.RLock()
defer reg.RUnlock()
for _, res := range reg.resources {
res.Lock()
res.selectVersion()
res.Unlock()
}
}
// GetSelectedVersions returns a list of the currently selected versions.
func (reg *ResourceRegistry) GetSelectedVersions() (versions map[string]string) {
reg.RLock()
defer reg.RUnlock()
for _, res := range reg.resources {
res.Lock()
versions[res.Identifier] = res.SelectedVersion.VersionNumber
res.Unlock()
}
return
}
// Purge deletes old updates, retaining a certain amount, specified by the keep parameter. Will at least keep 2 updates per resource.
func (reg *ResourceRegistry) Purge(keep int) {
reg.RLock()
defer reg.RUnlock()
for _, res := range reg.resources {
res.Purge(keep)
}
}
// Cleanup removes temporary files.
func (reg *ResourceRegistry) Cleanup() error {
// delete download tmp dir
return os.RemoveAll(reg.tmpDir.Path)
}

38
updater/registry_test.go Normal file
View file

@ -0,0 +1,38 @@
package updater
import (
"io/ioutil"
"os"
"testing"
"github.com/safing/portbase/utils"
)
var (
registry *ResourceRegistry
)
func TestMain(m *testing.M) {
// setup
tmpDir, err := ioutil.TempDir("", "ci-portmaster-")
if err != nil {
panic(err)
}
registry = &ResourceRegistry{
Beta: true,
DevMode: true,
Online: true,
}
err = registry.Initialize(utils.NewDirStructure(tmpDir, 0777))
if err != nil {
panic(err)
}
// run
// call flag.Parse() here if TestMain uses flags
ret := m.Run()
// teardown
os.RemoveAll(tmpDir)
os.Exit(ret)
}

330
updater/resource.go Normal file
View file

@ -0,0 +1,330 @@
package updater
import (
"errors"
"os"
"path/filepath"
"sort"
"strings"
"sync"
"github.com/safing/portbase/log"
semver "github.com/hashicorp/go-version"
)
// Resource represents a resource (via an identifier) and multiple file versions.
type Resource struct {
sync.Mutex
registry *ResourceRegistry
notifier *notifier
Identifier string
Versions []*ResourceVersion
ActiveVersion *ResourceVersion
SelectedVersion *ResourceVersion
ForceDownload bool
}
// ResourceVersion represents a single version of a resource.
type ResourceVersion struct {
resource *Resource
VersionNumber string
semVer *semver.Version
Available bool
StableRelease bool
BetaRelease bool
Blacklisted bool
}
// Len is the number of elements in the collection. (sort.Interface for Versions)
func (res *Resource) Len() int {
return len(res.Versions)
}
// Less reports whether the element with index i should sort before the element with index j. (sort.Interface for Versions)
func (res *Resource) Less(i, j int) bool {
return res.Versions[i].semVer.GreaterThan(res.Versions[j].semVer)
}
// Swap swaps the elements with indexes i and j. (sort.Interface for Versions)
func (res *Resource) Swap(i, j int) {
res.Versions[i], res.Versions[j] = res.Versions[j], res.Versions[i]
}
// available returns whether any version of the resource is available.
func (res *Resource) available() bool {
for _, rv := range res.Versions {
if rv.Available {
return true
}
}
return false
}
func (reg *ResourceRegistry) newResource(identifier string) *Resource {
return &Resource{
registry: reg,
Identifier: identifier,
Versions: make([]*ResourceVersion, 0, 1),
}
}
// AddVersion adds a resource version to a resource.
func (res *Resource) AddVersion(version string, available, stableRelease, betaRelease bool) error {
res.Lock()
defer res.Unlock()
// reset stable or beta release flags
if stableRelease || betaRelease {
for _, rv := range res.Versions {
if stableRelease {
rv.StableRelease = false
}
if betaRelease {
rv.BetaRelease = false
}
}
}
var rv *ResourceVersion
// check for existing version
for _, possibleMatch := range res.Versions {
if possibleMatch.VersionNumber == version {
rv = possibleMatch
break
}
}
// create new version if none found
if rv == nil {
// parse to semver
sv, err := semver.NewVersion(version)
if err != nil {
return err
}
rv = &ResourceVersion{
resource: res,
VersionNumber: version,
semVer: sv,
}
res.Versions = append(res.Versions, rv)
}
// set flags
if available {
rv.Available = true
}
if stableRelease {
rv.StableRelease = true
}
if betaRelease {
rv.BetaRelease = true
}
return nil
}
// GetFile returns the selected version as a *File.
func (res *Resource) GetFile() *File {
res.Lock()
defer res.Unlock()
// check for notifier
if res.notifier == nil {
// create new notifier
res.notifier = newNotifier()
}
// check if version is selected
if res.SelectedVersion == nil {
res.selectVersion()
}
// create file
return &File{
resource: res,
version: res.SelectedVersion,
notifier: res.notifier,
versionedPath: res.SelectedVersion.versionedPath(),
storagePath: res.SelectedVersion.storagePath(),
}
}
func (res *Resource) selectVersion() {
sort.Sort(res)
// export after we finish
defer func() {
if res.ActiveVersion != nil && // resource has already been used
res.SelectedVersion != res.ActiveVersion && // new selected version does not match previously selected version
res.notifier != nil {
res.notifier.markAsUpgradeable()
res.notifier = nil
}
res.registry.notifyOfChanges()
}()
if len(res.Versions) == 0 {
res.SelectedVersion = nil // TODO: find a better way
return
}
// Target selection
// 1) Dev release if dev mode is active and ignore blacklisting
if res.registry.DevMode {
// get last element
rv := res.Versions[len(res.Versions)-1]
// check if it's a dev version
if rv.VersionNumber == "0" && rv.Available {
res.SelectedVersion = rv
return
}
}
// 2) Beta release if beta is active
if res.registry.Beta {
for _, rv := range res.Versions {
if rv.BetaRelease {
if !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
res.SelectedVersion = rv
return
}
break
}
}
}
// 3) Stable release
for _, rv := range res.Versions {
if rv.StableRelease {
if !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
res.SelectedVersion = rv
return
}
break
}
}
// 4) Latest stable release
for _, rv := range res.Versions {
if !strings.HasSuffix(rv.VersionNumber, "b") && !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
res.SelectedVersion = rv
return
}
}
// 5) Latest of any type
for _, rv := range res.Versions {
if !rv.Blacklisted && (rv.Available || rv.resource.registry.Online) {
res.SelectedVersion = rv
return
}
}
// 6) Default to newest
res.SelectedVersion = res.Versions[0]
}
// Blacklist blacklists the specified version and selects a new version.
func (res *Resource) Blacklist(version string) error {
res.Lock()
defer res.Unlock()
// count already blacklisted entries
valid := 0
for _, rv := range res.Versions {
if rv.VersionNumber == "0" {
continue // ignore dev versions
}
if !rv.Blacklisted {
valid++
}
}
if valid <= 1 {
return errors.New("cannot blacklist last version") // last one, cannot blacklist!
}
// find version and blacklist
for _, rv := range res.Versions {
if rv.VersionNumber == version {
// blacklist and update
rv.Blacklisted = true
res.selectVersion()
return nil
}
}
return errors.New("could not find version")
}
// Purge deletes old updates, retaining a certain amount, specified by the keep parameter. Will at least keep 2 updates per resource. After purging, new versions will be selected.
func (res *Resource) Purge(keep int) {
res.Lock()
defer res.Unlock()
// safeguard
if keep < 2 {
keep = 2
}
// keep versions
var validVersions int
var skippedActiveVersion bool
var skippedSelectedVersion bool
var purgeFrom int
for i, rv := range res.Versions {
// continue to purging?
if validVersions >= keep && // skip at least <keep> versions
skippedActiveVersion && // skip until active version
skippedSelectedVersion { // skip until selected version
purgeFrom = i
break
}
// keep active version
if !skippedActiveVersion && rv == res.ActiveVersion {
skippedActiveVersion = true
}
// keep selected version
if !skippedSelectedVersion && rv == res.SelectedVersion {
skippedSelectedVersion = true
}
// count valid (not blacklisted) versions
if !rv.Blacklisted {
validVersions++
}
}
// check if there is anything to purge
if purgeFrom < keep || purgeFrom > len(res.Versions) {
return
}
// purge phase
for _, rv := range res.Versions[purgeFrom:] {
// delete
err := os.Remove(rv.storagePath())
if err != nil {
log.Warningf("%s: failed to purge old resource %s: %s", res.registry.Name, rv.storagePath(), err)
}
}
// remove entries of deleted files
res.Versions = res.Versions[purgeFrom:]
res.selectVersion()
}
func (rv *ResourceVersion) versionedPath() string {
return GetVersionedPath(rv.resource.Identifier, rv.VersionNumber)
}
func (rv *ResourceVersion) storagePath() string {
return filepath.Join(rv.resource.registry.storageDir.Path, filepath.FromSlash(rv.versionedPath()))
}

81
updater/resource_test.go Normal file
View file

@ -0,0 +1,81 @@
package updater
import (
"testing"
)
func TestVersionSelection(t *testing.T) {
res := registry.newResource("test/a")
err := res.AddVersion("1.2.3", true, true, false)
if err != nil {
t.Fatal(err)
}
err = res.AddVersion("1.2.4b", true, false, true)
if err != nil {
t.Fatal(err)
}
err = res.AddVersion("1.2.2", true, false, false)
if err != nil {
t.Fatal(err)
}
err = res.AddVersion("1.2.5", false, true, false)
if err != nil {
t.Fatal(err)
}
err = res.AddVersion("0", true, false, false)
if err != nil {
t.Fatal(err)
}
registry.Online = true
registry.Beta = true
registry.DevMode = true
res.selectVersion()
if res.SelectedVersion.VersionNumber != "0" {
t.Errorf("selected version should be 0, not %s", res.SelectedVersion.VersionNumber)
}
registry.DevMode = false
res.selectVersion()
if res.SelectedVersion.VersionNumber != "1.2.4b" {
t.Errorf("selected version should be 1.2.4b, not %s", res.SelectedVersion.VersionNumber)
}
registry.Beta = false
res.selectVersion()
if res.SelectedVersion.VersionNumber != "1.2.5" {
t.Errorf("selected version should be 1.2.5, not %s", res.SelectedVersion.VersionNumber)
}
registry.Online = false
res.selectVersion()
if res.SelectedVersion.VersionNumber != "1.2.3" {
t.Errorf("selected version should be 1.2.3, not %s", res.SelectedVersion.VersionNumber)
}
f123 := res.GetFile()
f123.markActiveWithLocking()
err = res.Blacklist("1.2.3")
if err != nil {
t.Fatal(err)
}
if res.SelectedVersion.VersionNumber != "1.2.2" {
t.Errorf("selected version should be 1.2.2, not %s", res.SelectedVersion.VersionNumber)
}
if !f123.UpgradeAvailable() {
t.Error("upgrade should be available (flag)")
}
select {
case <-f123.WaitForAvailableUpgrade():
default:
t.Error("upgrade should be available (chan)")
}
t.Logf("resource: %+v", res)
for _, rv := range res.Versions {
t.Logf("version %s: %+v", rv.VersionNumber, rv)
}
}

159
updater/storage.go Normal file
View file

@ -0,0 +1,159 @@
package updater
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
"github.com/safing/portbase/log"
"github.com/safing/portbase/utils"
)
// ScanStorage scans root within the storage dir and adds found resources to the registry. If an error occurred, it is logged and the last error is returned. Everything that was found despite errors is added to the registry anyway. Leave root empty to scan the full storage dir.
func (reg *ResourceRegistry) ScanStorage(root string) error {
var lastError error
// prep root
if root == "" {
root = reg.storageDir.Path
} else {
var err error
root, err = filepath.Abs(root)
if err != nil {
return err
}
if !strings.HasPrefix(root, reg.storageDir.Path) {
return errors.New("supplied scan root path not within storage")
}
}
// walk fs
_ = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if err != nil {
lastError = fmt.Errorf("%s: could not read %s: %s", reg.Name, path, err)
log.Warning(lastError.Error())
return nil
}
// get relative path to storage
relativePath, err := filepath.Rel(reg.storageDir.Path, path)
if err != nil {
lastError = fmt.Errorf("%s: could not get relative path of %s: %s", reg.Name, path, err)
log.Warning(lastError.Error())
return nil
}
// ignore files in tmp dir
if strings.HasPrefix(relativePath, reg.tmpDir.Path) {
return nil
}
// convert to identifier and version
relativePath = filepath.ToSlash(relativePath)
identifier, version, ok := GetIdentifierAndVersion(relativePath)
if !ok {
// file does not conform to format
return nil
}
// save
err = reg.AddResource(identifier, version, true, false, false)
if err != nil {
lastError = fmt.Errorf("%s: could not get add resource %s v%s: %s", reg.Name, identifier, version, err)
log.Warning(lastError.Error())
}
return nil
})
return lastError
}
// LoadIndexes loads the current release indexes from disk and will fetch a new version if not available and online.
func (reg *ResourceRegistry) LoadIndexes() error {
err := reg.loadIndexFile("stable.json", true, false)
if err != nil {
err = reg.downloadIndex("stable.json", true, false)
if err != nil {
return err
}
}
err = reg.loadIndexFile("beta.json", false, true)
if err != nil {
err = reg.downloadIndex("beta.json", false, true)
if err != nil {
return err
}
}
return nil
}
func (reg *ResourceRegistry) loadIndexFile(name string, stableRelease, betaRelease bool) error {
data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, name))
if err != nil {
return err
}
releases := make(map[string]string)
err = json.Unmarshal(data, &releases)
if err != nil {
return err
}
if len(releases) == 0 {
return fmt.Errorf("%s is empty", name)
}
err = reg.AddResources(releases, false, stableRelease, betaRelease)
if err != nil {
log.Warningf("%s: failed to add resource: %s", reg.Name, err)
}
return nil
}
// CreateSymlinks creates a directory structure with unversions symlinks to the given updates list.
func (reg *ResourceRegistry) CreateSymlinks(symlinkRoot *utils.DirStructure) error {
err := os.RemoveAll(symlinkRoot.Path)
if err != nil {
return fmt.Errorf("failed to wipe symlink root: %s", err)
}
err = symlinkRoot.Ensure()
if err != nil {
return fmt.Errorf("failed to create symlink root: %s", err)
}
reg.RLock()
defer reg.RUnlock()
for _, res := range reg.resources {
if res.SelectedVersion == nil {
return fmt.Errorf("no selected version available for %s", res.Identifier)
}
targetPath := res.SelectedVersion.storagePath()
linkPath := filepath.Join(symlinkRoot.Path, filepath.FromSlash(res.Identifier))
linkPathDir := filepath.Dir(linkPath)
err = symlinkRoot.EnsureAbsPath(linkPathDir)
if err != nil {
return fmt.Errorf("failed to create dir for link: %s", err)
}
relativeTargetPath, err := filepath.Rel(linkPathDir, targetPath)
if err != nil {
return fmt.Errorf("failed to get relative target path: %s", err)
}
err = os.Symlink(relativeTargetPath, linkPath)
if err != nil {
return fmt.Errorf("failed to link %s: %s", res.Identifier, err)
}
}
return nil
}

68
updater/storage_test.go Normal file
View file

@ -0,0 +1,68 @@
package updater
/*
func testLoadLatestScope(t *testing.T, basePath, filePath, expectedIdentifier, expectedVersion string) {
fullPath := filepath.Join(basePath, filePath)
// create dir
dirPath := filepath.Dir(fullPath)
err := os.MkdirAll(dirPath, 0755)
if err != nil {
t.Fatalf("could not create test dir: %s\n", err)
return
}
// touch file
err = ioutil.WriteFile(fullPath, []byte{}, 0644)
if err != nil {
t.Fatalf("could not create test file: %s\n", err)
return
}
// run loadLatestScope
latest, err := ScanForLatest(basePath, true)
if err != nil {
t.Errorf("could not update latest: %s\n", err)
return
}
for key, val := range latest {
localUpdates[key] = val
}
// test result
version, ok := localUpdates[expectedIdentifier]
if !ok {
t.Errorf("identifier %s not in map", expectedIdentifier)
t.Errorf("current map: %v", localUpdates)
}
if version != expectedVersion {
t.Errorf("unexpected version for %s: %s", filePath, version)
}
}
func TestLoadLatestScope(t *testing.T) {
updatesLock.Lock()
defer updatesLock.Unlock()
tmpDir, err := ioutil.TempDir("", "testing_")
if err != nil {
t.Fatalf("could not create test dir: %s\n", err)
return
}
defer os.RemoveAll(tmpDir)
testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "1.2.3")
testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4b.zip", "all/ui/assets.zip", "1.2.4b")
testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-5.zip", "all/ui/assets.zip", "1.2.5")
testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "1.3.4")
testLoadLatestScope(t, tmpDir, "all/ui/assets_v2-3-4.zip", "all/ui/assets.zip", "2.3.4")
testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "2.3.4")
testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "2.3.4")
testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "2.3.4")
testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "1.2.3")
testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v2-1-1", "os_platform/portmaster/portmaster", "2.1.1")
testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "2.1.1")
}
*/

127
updater/updating.go Normal file
View file

@ -0,0 +1,127 @@
package updater
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"github.com/safing/portbase/utils"
"github.com/safing/portbase/log"
)
// UpdateIndexes downloads the current update indexes.
func (reg *ResourceRegistry) UpdateIndexes() error {
err := reg.downloadIndex("stable.json", true, false)
if err != nil {
return err
}
return reg.downloadIndex("beta.json", false, true)
}
func (reg *ResourceRegistry) downloadIndex(name string, stableRelease, betaRelease bool) error {
var err error
var data []byte
// download new index
for tries := 0; tries < 3; tries++ {
data, err = reg.fetchData(name, tries)
if err == nil {
break
}
}
if err != nil {
return fmt.Errorf("failed to download index %s: %s", name, err)
}
// parse
new := make(map[string]string)
err = json.Unmarshal(data, &new)
if err != nil {
return fmt.Errorf("failed to parse index %s: %s", name, err)
}
// check for content
if len(new) == 0 {
return fmt.Errorf("index %s is empty", name)
}
// add resources to registry
_ = reg.AddResources(new, false, stableRelease, betaRelease)
// save index
err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, name), data, 0644)
if err != nil {
log.Warningf("%s: failed to save updated index %s: %s", reg.Name, name, err)
}
log.Infof("%s: updated index %s", reg.Name, name)
return nil
}
// DownloadUpdates checks if updates are available and downloads updates of used components.
func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error {
// create list of downloads
var toUpdate []*ResourceVersion
reg.RLock()
for _, res := range reg.resources {
res.Lock()
// check if we want to download
if res.ActiveVersion != nil || // resource is currently being used
res.available() || // resource was used in the past
utils.StringInSlice(reg.MandatoryUpdates, res.Identifier) { // resource is mandatory
// add all non-available and eligible versions to update queue
for _, rv := range res.Versions {
if !rv.Available && (rv.StableRelease || reg.Beta && rv.BetaRelease) {
toUpdate = append(toUpdate, rv)
}
}
}
res.Unlock()
}
reg.RUnlock()
// nothing to update
if len(toUpdate) == 0 {
log.Infof("%s: everything up to date", reg.Name)
return nil
}
// check download dir
err := reg.tmpDir.Ensure()
if err != nil {
return fmt.Errorf("could not prepare tmp directory for download: %s", err)
}
// download updates
log.Infof("%s: starting to download %d updates", reg.Name, len(toUpdate))
for _, rv := range toUpdate {
for tries := 0; tries < 3; tries++ {
err = reg.fetchFile(rv, tries)
if err == nil {
break
}
}
if err != nil {
log.Warningf("%s: failed to download %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err)
}
}
log.Infof("%s: finished downloading updates", reg.Name)
// remove tmp folder after we are finished
err = os.RemoveAll(reg.tmpDir.Path)
if err != nil {
log.Tracef("%s: failed to remove tmp dir %s after downloading updates: %s", reg.Name, reg.tmpDir.Path, err)
}
return nil
}

1
updater/uptool/.gitignore vendored Normal file
View file

@ -0,0 +1 @@
uptool

37
updater/uptool/root.go Normal file
View file

@ -0,0 +1,37 @@
package main
import (
"os"
"path/filepath"
"github.com/safing/portbase/updater"
"github.com/safing/portbase/utils"
"github.com/spf13/cobra"
)
var registry *updater.ResourceRegistry
var rootCmd = &cobra.Command{
Use: "uptool",
Short: "helper tool for the update process",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
return cmd.Usage()
},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
absPath, err := filepath.Abs(args[0])
if err != nil {
return err
}
registry = &updater.ResourceRegistry{}
return registry.Initialize(utils.NewDirStructure(absPath, 0755))
},
SilenceUsage: true,
}
func main() {
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
}
}

80
updater/uptool/scan.go Normal file
View file

@ -0,0 +1,80 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"strings"
"github.com/spf13/cobra"
)
func init() {
rootCmd.AddCommand(scanCmd)
}
var scanCmd = &cobra.Command{
Use: "scan",
Short: "Scan the specified directory and print the result",
Args: cobra.ExactArgs(1),
RunE: scan,
}
func scan(cmd *cobra.Command, args []string) error {
err := scanStorage()
if err != nil {
return err
}
// export beta
data, err := json.MarshalIndent(exportSelected(true), "", " ")
if err != nil {
return err
}
// print
fmt.Println("beta:")
fmt.Println(string(data))
// export stable
data, err = json.MarshalIndent(exportSelected(false), "", " ")
if err != nil {
return err
}
// print
fmt.Println("\nstable:")
fmt.Println(string(data))
return nil
}
func scanStorage() error {
files, err := ioutil.ReadDir(registry.StorageDir().Path)
if err != nil {
return err
}
// scan "all" and all "os_platform" dirs
for _, file := range files {
if file.IsDir() && (file.Name() == "all" || strings.Contains(file.Name(), "_")) {
err := registry.ScanStorage(filepath.Join(registry.StorageDir().Path, file.Name()))
if err != nil {
return err
}
}
}
return nil
}
func exportSelected(beta bool) map[string]string {
registry.SetBeta(beta)
registry.SelectVersions()
export := registry.Export()
versions := make(map[string]string)
for _, rv := range export {
versions[rv.Identifier] = rv.SelectedVersion.VersionNumber
}
return versions
}

64
updater/uptool/update.go Normal file
View file

@ -0,0 +1,64 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"path/filepath"
"github.com/spf13/cobra"
)
func init() {
rootCmd.AddCommand(updateCmd)
}
var updateCmd = &cobra.Command{
Use: "update",
Short: "Update scans the specified directory and registry the index and symlink structure",
Args: cobra.ExactArgs(1),
RunE: update,
}
func update(cmd *cobra.Command, args []string) error {
err := scanStorage()
if err != nil {
return err
}
// export beta
data, err := json.MarshalIndent(exportSelected(true), "", " ")
if err != nil {
return err
}
// print
fmt.Println("beta:")
fmt.Println(string(data))
// write index
err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "beta.json"), data, 0755)
if err != nil {
return err
}
// export stable
data, err = json.MarshalIndent(exportSelected(false), "", " ")
if err != nil {
return err
}
// print
fmt.Println("\nstable:")
fmt.Println(string(data))
// write index
err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "stable.json"), data, 0755)
if err != nil {
return err
}
// create symlinks
err = registry.CreateSymlinks(registry.StorageDir().ChildDir("latest", 0755))
if err != nil {
return err
}
fmt.Println("\nstable symlinks created")
return nil
}