Add updates module and fix issues

firewall/master.go

@@ -1,6 +1,7 @@
 package firewall
 
 import (
+	"fmt"
 	"os"
 	"strings"
 
@@ -57,14 +58,14 @@ func DecideOnConnectionBeforeIntel(connection *network.Connection, fqdn string)
 	}
 
 	// check domain list
-	permitted, ok := profileSet.CheckDomain(fqdn)
+	permitted, reason, ok := profileSet.CheckEndpoint(fqdn, 0, 0, false)
 	if ok {
 		if permitted {
-			log.Infof("firewall: accepting connection %s, domain is whitelisted", connection)
-			connection.Accept("domain is whitelisted")
+			log.Infof("firewall: accepting connection %s, endpoint is whitelisted: %s", connection, reason)
+			connection.Accept(fmt.Sprintf("endpoint is whitelisted: %s", reason))
 		} else {
-			log.Infof("firewall: denying connection %s, domain is blacklisted", connection)
-			connection.Deny("domain is blacklisted")
+			log.Infof("firewall: denying connection %s, endpoint is blacklisted", connection)
+			connection.Deny("endpoint is blacklisted")
 		}
 		return
 	}
@@ -207,6 +208,8 @@ func DecideOnConnection(connection *network.Connection, pkt packet.Packet) {
 			connection.Deny("peer to peer connections (to an IP) not allowed")
 			return
 		}
+	default:
+
 	}
 
 	// check network scope
@@ -269,6 +272,13 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet
 	// Profile.ConnectPorts
 	// Profile.ListenPorts
 
+	// grant self
+	if connection.Process().Pid == os.Getpid() {
+		log.Infof("firewall: granting own link %s", connection)
+		connection.Accept("")
+		return
+	}
+
 	// check if there is a profile
 	profileSet := connection.Process().ProfileSet()
 	if profileSet == nil {
@@ -278,7 +288,18 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet
 	}
 	profileSet.Update(status.CurrentSecurityLevel())
 
-	// get remote Port
+	// get host
+	var domainOrIP string
+	switch {
+	case strings.HasSuffix(connection.Domain, "."):
+		domainOrIP = connection.Domain
+	case connection.Direction:
+		domainOrIP = pkt.GetIPHeader().Src.String()
+	default:
+		domainOrIP = pkt.GetIPHeader().Dst.String()
+	}
+
+	// get protocol / destination port
 	protocol := pkt.GetIPHeader().Protocol
 	var dstPort uint16
 	tcpUDPHeader := pkt.GetTCPUDPHeader()
@@ -286,12 +307,12 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet
 		dstPort = tcpUDPHeader.DstPort
 	}
 
-	// check port list
-	permitted, ok := profileSet.CheckPort(connection.Direction, uint8(protocol), dstPort)
+	// check endpoints list
+	permitted, reason, ok := profileSet.CheckEndpoint(domainOrIP, uint8(protocol), dstPort, connection.Direction)
 	if ok {
 		if permitted {
-			log.Infof("firewall: accepting link %s", link)
-			link.Accept("port whitelisted")
+			log.Infof("firewall: accepting link %s, endpoint is whitelisted: %s", link, reason)
+			link.Accept(fmt.Sprintf("port whitelisted: %s", reason))
 		} else {
 			log.Infof("firewall: denying link %s: port %d is blacklisted", link, dstPort)
 			link.Deny("port blacklisted")
@@ -301,16 +322,16 @@ func DecideOnLink(connection *network.Connection, link *network.Link, pkt packet
 
 	switch profileSet.GetProfileMode() {
 	case profile.Whitelist:
-		log.Infof("firewall: denying link %s: port %d is not whitelisted", link, dstPort)
-		link.Deny("port is not whitelisted")
+		log.Infof("firewall: denying link %s: endpoint %d is not whitelisted", link, dstPort)
+		link.Deny("endpoint is not whitelisted")
 		return
 	case profile.Prompt:
-		log.Infof("firewall: accepting link %s: port %d is blacklisted", link, dstPort)
-		link.Accept("port permitted (prompting is not yet implemented)")
+		log.Infof("firewall: accepting link %s: endpoint %d is blacklisted", link, dstPort)
+		link.Accept("endpoint permitted (prompting is not yet implemented)")
 		return
 	case profile.Blacklist:
-		log.Infof("firewall: accepting link %s: port %d is not blacklisted", link, dstPort)
-		link.Accept("port is not blacklisted")
+		log.Infof("firewall: accepting link %s: endpoint %d is not blacklisted", link, dstPort)
+		link.Accept("endpoint is not blacklisted")
 		return
 	}
 

main.go

@@ -20,6 +20,7 @@ import (
 	_ "github.com/Safing/portbase/database/storage/badger"
 	_ "github.com/Safing/portmaster/firewall"
 	_ "github.com/Safing/portmaster/nameserver"
+	_ "github.com/Safing/portmaster/ui"
 )
 
 var (

process/database.go

@@ -68,7 +68,7 @@ func (p *Process) Save() {
 // Delete deletes a process from the storage and propagates the change.
 func (p *Process) Delete() {
 	p.Lock()
-	defer p.Lock()
+	defer p.Unlock()
 
 	processesLock.Lock()
 	delete(processes, p.Pid)
@@ -79,7 +79,10 @@ func (p *Process) Delete() {
 		go dbController.PushUpdate(p)
 	}
 
-	profile.DeactivateProfileSet(p.profileSet)
+	// TODO: this should not be necessary, as processes should always have a profileSet.
+	if p.profileSet != nil {
+		profile.DeactivateProfileSet(p.profileSet)
+	}
 }
 
 // CleanProcessStorage cleans the storage from old processes.

profile/endpoints.go

@@ -56,8 +56,10 @@ func (e Endpoints) Check(domainOrIP string, protocol uint8, port uint16, checkRe
 	isDomain := strings.HasSuffix(domainOrIP, ".")
 
 	for _, entry := range e {
-		if ok, reason := entry.Matches(domainOrIP, protocol, port, isDomain, cachedGetDomainOfIP); ok {
-			return entry.Permit, reason, true
+		if entry != nil {
+			if ok, reason := entry.Matches(domainOrIP, protocol, port, isDomain, cachedGetDomainOfIP); ok {
+				return entry.Permit, reason, true
+			}
 		}
 	}
 

profile/profile.go

@@ -57,6 +57,7 @@ func New() *Profile {
 	}
 }
 
+// MakeProfileKey creates the correct key for a profile with the given namespace and ID.
 func MakeProfileKey(namespace, ID string) string {
 	return fmt.Sprintf("core:profiles/%s/%s", namespace, ID)
 }

profile/specialprofiles.go

@@ -47,13 +47,15 @@ func getSpecialProfile(ID string) (*Profile, error) {
 
 func ensureServiceEndpointsDenyAll(p *Profile) (changed bool) {
 	for _, ep := range p.ServiceEndpoints {
-		if ep.DomainOrIP == "" &&
-			ep.Wildcard == true &&
-			ep.Protocol == 0 &&
-			ep.StartPort == 0 &&
-			ep.EndPort == 0 &&
-			ep.Permit == false {
-			return false
+		if ep != nil {
+			if ep.DomainOrIP == "" &&
+				ep.Wildcard == true &&
+				ep.Protocol == 0 &&
+				ep.StartPort == 0 &&
+				ep.EndPort == 0 &&
+				ep.Permit == false {
+				return false
+			}
 		}
 	}
 

ui/launch.go

@@ -0,0 +1,66 @@
+package ui
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"os/exec"
+	"runtime"
+
+	"github.com/Safing/portbase/modules"
+	"github.com/Safing/portmaster/updates"
+)
+
+var (
+	launchUI bool
+)
+
+func init() {
+	flag.BoolVar(&launchUI, "ui", false, "launch user interface and exit")
+}
+
+func launchUIByFlag() error {
+	if !launchUI {
+		return nil
+	}
+
+	err := updates.ReloadLatest()
+	if err != nil {
+		return err
+	}
+
+	osAndPlatform := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
+
+	switch osAndPlatform {
+	case "linux_amd64":
+
+		file, err := updates.GetPlatformFile("app/portmaster-ui")
+		if err != nil {
+			return fmt.Errorf("ui currently not available: %s - you may need to first start portmaster and wait for it to fetch the update index", err)
+		}
+
+		// check permission
+		info, err := os.Stat(file.Path())
+		if info.Mode() != 0755 {
+			fmt.Printf("%v\n", info.Mode())
+			err := os.Chmod(file.Path(), 0755)
+			if err != nil {
+				return fmt.Errorf("failed to set exec permissions on %s: %s", file.Path(), err)
+			}
+		}
+
+		// exec
+		cmd := exec.Command(file.Path())
+		err = cmd.Start()
+		if err != nil {
+			return fmt.Errorf("failed to start ui: %s", err)
+		}
+
+		// gracefully exit portmaster
+		return modules.ErrCleanExit
+
+	default:
+		return errors.New("this os/platform is no UI support yet")
+	}
+}

ui/module.go

@@ -5,13 +5,14 @@ import (
 )
 
 func init() {
-	modules.Register("ui", prep, start, stop, "database", "api")
+	modules.Register("ui", prep, nil, nil, "updates", "api")
 }
 
 func prep() error {
-	return nil
-}
+	err := launchUIByFlag()
+	if err != nil {
+		return err
+	}
 
-func stop() error {
-	return nil
+	return registerRoutes()
 }

ui/serve.go

@@ -6,16 +6,16 @@ import (
 	"mime"
 	"net/http"
 	"net/url"
-	"path"
 	"path/filepath"
+	"strings"
 	"sync"
 
 	resources "github.com/cookieo9/resources-go"
 	"github.com/gorilla/mux"
 
 	"github.com/Safing/portbase/api"
-	"github.com/Safing/portbase/database"
 	"github.com/Safing/portbase/log"
+	"github.com/Safing/portmaster/updates"
 )
 
 var (
@@ -25,66 +25,80 @@ var (
 	assetsLock sync.RWMutex
 )
 
-func start() error {
-	basePath := path.Join(database.GetDatabaseRoot(), "updates", "files", "apps")
-
-	serveUIRouter := mux.NewRouter()
-	serveUIRouter.HandleFunc("/assets/{resPath:[a-zA-Z0-9/\\._-]+}", ServeAssets(basePath))
-	serveUIRouter.HandleFunc("/app/{appName:[a-z]+}/", ServeApps(basePath))
-	serveUIRouter.HandleFunc("/app/{appName:[a-z]+}/{resPath:[a-zA-Z0-9/\\._-]+}", ServeApps(basePath))
-	serveUIRouter.HandleFunc("/", RedirectToControl)
-
-	api.RegisterAdditionalRoute("/assets/", serveUIRouter)
-	api.RegisterAdditionalRoute("/app/", serveUIRouter)
-	api.RegisterAdditionalRoute("/", serveUIRouter)
+func registerRoutes() error {
+	api.RegisterHandleFunc("/assets/{resPath:[a-zA-Z0-9/\\._-]+}", ServeBundle("assets")).Methods("GET", "HEAD")
+	api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}", redirAddSlash).Methods("GET", "HEAD")
+	api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}/", ServeBundle("")).Methods("GET", "HEAD")
+	api.RegisterHandleFunc("/ui/modules/{moduleName:[a-z]+}/{resPath:[a-zA-Z0-9/\\._-]+}", ServeBundle("")).Methods("GET", "HEAD")
+	api.RegisterHandleFunc("/", RedirectToBase)
 
 	return nil
 }
 
-// ServeApps serves app files.
-func ServeApps(basePath string) func(w http.ResponseWriter, r *http.Request) {
+// ServeBundle serves bundles.
+func ServeBundle(defaultModuleName string) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 
+		// log.Tracef("ui: request for %s", r.RequestURI)
+
 		vars := mux.Vars(r)
-		appName, ok := vars["appName"]
+		moduleName, ok := vars["moduleName"]
 		if !ok {
-			http.Error(w, "missing app name", http.StatusBadRequest)
-			return
+			moduleName = defaultModuleName
+			if moduleName == "" {
+				http.Error(w, "missing module name", http.StatusBadRequest)
+				return
+			}
 		}
 
 		resPath, ok := vars["resPath"]
-		if !ok {
-			http.Error(w, "missing resource path", http.StatusBadRequest)
-			return
+		if !ok || strings.HasSuffix(resPath, "/") {
+			resPath = "index.html"
 		}
 
 		appsLock.RLock()
-		bundle, ok := apps[appName]
+		bundle, ok := apps[moduleName]
 		appsLock.RUnlock()
 		if ok {
-			ServeFileFromBundle(w, r, bundle, resPath)
+			ServeFileFromBundle(w, r, moduleName, bundle, resPath)
 			return
 		}
 
-		newBundle, err := resources.OpenZip(path.Join(basePath, fmt.Sprintf("%s.zip", appName)))
+		// get file from update system
+		zipFile, err := updates.GetFile(fmt.Sprintf("ui/modules/%s.zip", moduleName))
 		if err != nil {
+			if err == updates.ErrNotFound {
+				log.Tracef("ui: requested module %s does not exist", moduleName)
+				http.Error(w, err.Error(), http.StatusNotFound)
+			} else {
+				log.Tracef("ui: error loading module %s: %s", moduleName, err)
+				http.Error(w, err.Error(), http.StatusInternalServerError)
+			}
+			return
+		}
+
+		// open bundle
+		newBundle, err := resources.OpenZip(zipFile.Path())
+		if err != nil {
+			log.Tracef("ui: error prepping module %s: %s", moduleName, err)
 			http.Error(w, err.Error(), http.StatusInternalServerError)
 			return
 		}
 
 		bundle = &resources.BundleSequence{newBundle}
 		appsLock.Lock()
-		apps[appName] = bundle
+		apps[moduleName] = bundle
 		appsLock.Unlock()
 
-		ServeFileFromBundle(w, r, bundle, resPath)
+		ServeFileFromBundle(w, r, moduleName, bundle, resPath)
 	}
 }
 
 // ServeFileFromBundle serves a file from the given bundle.
-func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundle *resources.BundleSequence, path string) {
+func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundleName string, bundle *resources.BundleSequence, path string) {
 	readCloser, err := bundle.Open(path)
 	if err != nil {
+		log.Tracef("ui: error opening module %s: %s", bundleName, err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
@@ -110,45 +124,16 @@ func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundle *resourc
 	return
 }
 
-// ServeAssets serves global UI assets.
-func ServeAssets(basePath string) func(w http.ResponseWriter, r *http.Request) {
-	return func(w http.ResponseWriter, r *http.Request) {
-
-		vars := mux.Vars(r)
-		resPath, ok := vars["resPath"]
-		if !ok {
-			http.Error(w, "missing resource path", http.StatusBadRequest)
-			return
-		}
-
-		assetsLock.RLock()
-		bundle := assets
-		assetsLock.RUnlock()
-		if bundle != nil {
-			ServeFileFromBundle(w, r, bundle, resPath)
-		}
-
-		newBundle, err := resources.OpenZip(path.Join(basePath, "assets.zip"))
-		if err != nil {
-			http.Error(w, err.Error(), http.StatusInternalServerError)
-			return
-		}
-
-		bundle = &resources.BundleSequence{newBundle}
-		assetsLock.Lock()
-		assets = bundle
-		assetsLock.Unlock()
-
-		ServeFileFromBundle(w, r, bundle, resPath)
-	}
-}
-
-// RedirectToControl redirects the requests to the control app
-func RedirectToControl(w http.ResponseWriter, r *http.Request) {
-	u, err := url.Parse("/app/control")
+// RedirectToBase redirects the requests to the control app
+func RedirectToBase(w http.ResponseWriter, r *http.Request) {
+	u, err := url.Parse("/ui/modules/base/")
 	if err != nil {
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
 	http.Redirect(w, r, r.URL.ResolveReference(u).String(), http.StatusPermanentRedirect)
 }
+
+func redirAddSlash(w http.ResponseWriter, r *http.Request) {
+	http.Redirect(w, r, r.RequestURI+"/", http.StatusPermanentRedirect)
+}

updates/doc.go

@@ -0,0 +1,9 @@
+package updates
+
+// current paths:
+// all/ui/assets.zip
+// all/ui/modules/base.zip
+// all/ui/modules/settings.zip
+// all/ui/modules/profilemgr.zip
+// all/ui/modules/monitor.zip
+// linux_amd64/app/portmaster-ui

updates/fetch.go

@@ -0,0 +1,122 @@
+package updates
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+	"time"
+
+	"github.com/google/renameio"
+
+	"github.com/Safing/portbase/log"
+)
+
+var (
+	updateURLs = []string{
+		"https://updates.safing.io",
+	}
+)
+
+func fetchFile(realFilepath, updateFilepath string, tries int) error {
+	// backoff when retrying
+	if tries > 0 {
+		time.Sleep(time.Duration(tries*tries) * time.Second)
+	}
+
+	// create URL
+	downloadURL, err := joinURLandPath(updateURLs[tries%len(updateURLs)], updateFilepath)
+	if err != nil {
+		return fmt.Errorf("error build url (%s + %s): %s", updateURLs[tries%len(updateURLs)], updateFilepath, err)
+	}
+
+	// create destination dir
+	dirPath := filepath.Dir(realFilepath)
+	err = os.MkdirAll(dirPath, 0755)
+	if err != nil {
+		return fmt.Errorf("updates: could not create updates folder: %s", dirPath)
+	}
+
+	// open file for writing
+	atomicFile, err := renameio.TempFile(filepath.Join(updateStoragePath, "tmp"), realFilepath)
+	if err != nil {
+		return fmt.Errorf("updates: could not create temp file for download: %s", err)
+	}
+	defer atomicFile.Cleanup()
+
+	// start file download
+	resp, err := http.Get(downloadURL)
+	if err != nil {
+		return fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+	}
+	defer resp.Body.Close()
+
+	// download and write file
+	n, err := io.Copy(atomicFile, resp.Body)
+	if err != nil {
+		return fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+	}
+	if resp.ContentLength != n {
+		return fmt.Errorf("download unfinished, written %d out of %d bytes.", n, resp.ContentLength)
+	}
+
+	// finalize file
+	err = atomicFile.CloseAtomicallyReplace()
+	if err != nil {
+		return fmt.Errorf("updates: failed to finalize file %s: %s", realFilepath, err)
+	}
+	// set permissions
+	err = os.Chmod(realFilepath, 0644)
+	if err != nil {
+		log.Warningf("updates: failed to set permissions on downloaded file %s: %s", realFilepath, err)
+	}
+
+	log.Infof("update: fetched %s (stored to %s)", downloadURL, realFilepath)
+	return nil
+}
+
+func fetchData(downloadPath string, tries int) ([]byte, error) {
+	// backoff when retrying
+	if tries > 0 {
+		time.Sleep(time.Duration(tries*tries) * time.Second)
+	}
+
+	// create URL
+	downloadURL, err := joinURLandPath(updateURLs[tries%len(updateURLs)], downloadPath)
+	if err != nil {
+		return nil, fmt.Errorf("error build url (%s + %s): %s", updateURLs[tries%len(updateURLs)], downloadPath, err)
+	}
+
+	// start file download
+	resp, err := http.Get(downloadURL)
+	if err != nil {
+		return nil, fmt.Errorf("error fetching url (%s): %s", downloadURL, err)
+	}
+	defer resp.Body.Close()
+
+	// download and write file
+	buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
+	n, err := io.Copy(buf, resp.Body)
+	if err != nil {
+		return nil, fmt.Errorf("failed downloading %s: %s", downloadURL, err)
+	}
+	if resp.ContentLength != n {
+		return nil, fmt.Errorf("download unfinished, written %d out of %d bytes.", n, resp.ContentLength)
+	}
+
+	return buf.Bytes(), nil
+}
+
+func joinURLandPath(baseURL, urlPath string) (string, error) {
+	u, err := url.Parse(baseURL)
+	if err != nil {
+		return "", err
+	}
+
+	u.Path = path.Join(u.Path, urlPath)
+	return u.String(), nil
+}

updates/file.go

@@ -0,0 +1,46 @@
+package updates
+
+// File represents a file from the update system.
+type File struct {
+	filepath string
+	version  string
+	stable   bool
+}
+
+func newFile(filepath string, version string, stable bool) *File {
+	return &File{
+		filepath: filepath,
+		version:  version,
+		stable:   stable,
+	}
+}
+
+// Path returns the filepath of the file.
+func (f *File) Path() string {
+	return f.filepath
+}
+
+// Version returns the version of the file.
+func (f *File) Version() string {
+	return f.version
+}
+
+// Stable returns whether the file is from a stable release.
+func (f *File) Stable() bool {
+	return f.stable
+}
+
+// Open opens the file and returns the
+func (f *File) Open() {
+
+}
+
+// ReportError reports an error back to Safing. This will not automatically blacklist the file.
+func (f *File) ReportError() {
+
+}
+
+// Blacklist notifies the update system that this file is somehow broken, and should be ignored from now on.
+func (f *File) Blacklist() {
+
+}

updates/filename.go

@@ -0,0 +1,41 @@
+package updates
+
+import (
+	"fmt"
+	"regexp"
+	"strings"
+)
+
+var versionRegex = regexp.MustCompile("_v[0-9]+-[0-9]+-[0-9]+b?")
+
+func getIdentifierAndVersion(versionedPath string) (identifier, version string, ok bool) {
+	// extract version
+	rawVersion := versionRegex.FindString(versionedPath)
+	if rawVersion == "" {
+		return "", "", false
+	}
+
+	// replace - with . and trim _
+	version = strings.Replace(strings.TrimLeft(rawVersion, "_v"), "-", ".", -1)
+
+	// put together without version
+	i := strings.Index(versionedPath, rawVersion)
+	if i < 0 {
+		// extracted version not in string (impossible)
+		return "", "", false
+	}
+	return versionedPath[:i] + versionedPath[i+len(rawVersion):], version, true
+}
+
+func getVersionedPath(identifier, version string) (versionedPath string) {
+	// split in half
+	splittedFilePath := strings.SplitN(identifier, ".", 2)
+	// replace . with -
+	transformedVersion := strings.Replace(version, ".", "-", -1)
+
+	// put together
+	if len(splittedFilePath) == 1 {
+		return fmt.Sprintf("%s_v%s", splittedFilePath[0], transformedVersion)
+	}
+	return fmt.Sprintf("%s_v%s.%s", splittedFilePath[0], transformedVersion, splittedFilePath[1])
+}

updates/get.go

@@ -0,0 +1,77 @@
+package updates
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/Safing/portbase/log"
+)
+
+var (
+	ErrNotFound = errors.New("the requested file could not be found")
+)
+
+// GetPlatformFile returns the latest platform specific file identified by the given identifier.
+func GetPlatformFile(identifier string) (*File, error) {
+	identifier = filepath.Join(fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH), identifier)
+	// From https://golang.org/pkg/runtime/#GOARCH
+	// GOOS is the running program's operating system target: one of darwin, freebsd, linux, and so on.
+	// GOARCH is the running program's architecture target: one of 386, amd64, arm, s390x, and so on.
+	return loadOrFetchFile(identifier)
+}
+
+// GetFile returns the latest generic file identified by the given identifier.
+func GetFile(identifier string) (*File, error) {
+	identifier = filepath.Join("all", identifier)
+	return loadOrFetchFile(identifier)
+}
+
+func getLatestFilePath(identifier string) (versionedFilePath, version string, stable bool, ok bool) {
+	updatesLock.RLock()
+	version, ok = stableUpdates[identifier]
+	if !ok {
+		version, ok = latestUpdates[identifier]
+		if !ok {
+			log.Tracef("updates: file %s does not exist", identifier)
+			return "", "", false, false
+			// TODO: if in development mode, reload latest index to check for newly sideloaded updates
+			// err := reloadLatest()
+		}
+	}
+	updatesLock.RUnlock()
+
+	// TODO: Fix for stable release
+	return getVersionedPath(identifier, version), version, false, true
+}
+
+func loadOrFetchFile(identifier string) (*File, error) {
+	versionedFilePath, version, stable, ok := getLatestFilePath(identifier)
+	if !ok {
+		// TODO: if in development mode, search updates dir for sideloaded apps
+		return nil, ErrNotFound
+	}
+
+	// build final filepath
+	realFilePath := filepath.Join(updateStoragePath, versionedFilePath)
+	if _, err := os.Stat(realFilePath); err == nil {
+		// file exists
+		return newFile(realFilePath, version, stable), nil
+	}
+
+	// download file
+	log.Tracef("updates: starting download of %s", versionedFilePath)
+	var err error
+	for tries := 0; tries < 5; tries++ {
+		err := fetchFile(realFilePath, versionedFilePath, tries)
+		if err != nil {
+			log.Tracef("updates: failed to download %s: %s, retrying (%d)", versionedFilePath, err, tries+1)
+		} else {
+			return newFile(realFilePath, version, stable), nil
+		}
+	}
+	log.Warningf("updates: failed to download %s: %s", versionedFilePath, err)
+	return nil, err
+}

updates/get_test.go

@@ -0,0 +1,24 @@
+package updates
+
+import "testing"
+
+func testBuildVersionedFilePath(t *testing.T, identifier, version, expectedVersionedFilePath string) {
+	updatesLock.Lock()
+	stableUpdates[identifier] = version
+	// betaUpdates[identifier] = version
+	updatesLock.Unlock()
+
+	versionedFilePath, _, _, ok := getLatestFilePath(identifier)
+	if !ok {
+		t.Errorf("identifier %s should exist", identifier)
+	}
+	if versionedFilePath != expectedVersionedFilePath {
+		t.Errorf("unexpected versionedFilePath: %s", versionedFilePath)
+	}
+}
+
+func TestBuildVersionedFilePath(t *testing.T) {
+	testBuildVersionedFilePath(t, "path/to/asset.zip", "1.2.3", "path/to/asset_v1-2-3.zip")
+	testBuildVersionedFilePath(t, "path/to/asset.tar.gz", "1.2.3b", "path/to/asset_v1-2-3b.tar.gz")
+	testBuildVersionedFilePath(t, "path/to/asset", "1.2.3b", "path/to/asset_v1-2-3b")
+}

updates/latest.go

@@ -0,0 +1,141 @@
+package updates
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/Safing/portbase/log"
+)
+
+var (
+	stableUpdates = make(map[string]string)
+	betaUpdates   = make(map[string]string)
+	latestUpdates = make(map[string]string)
+	updatesLock   sync.RWMutex
+)
+
+// ReloadLatest reloads available updates from disk.
+func ReloadLatest() error {
+	newLatestUpdates := make(map[string]string)
+
+	// all
+	new, err1 := ScanForLatest(filepath.Join(updateStoragePath, "all"), false)
+	for key, val := range new {
+		newLatestUpdates[key] = val
+	}
+
+	// os_platform
+	new, err2 := ScanForLatest(filepath.Join(updateStoragePath, fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)), false)
+	for key, val := range new {
+		newLatestUpdates[key] = val
+	}
+
+	if err1 != nil && err2 != nil {
+		return fmt.Errorf("could not load latest update versions: %s, %s", err1, err2)
+	}
+
+	log.Tracef("updates: loading latest updates:")
+
+	for key, val := range newLatestUpdates {
+		log.Tracef("updates: %s v%s", key, val)
+	}
+
+	updatesLock.Lock()
+	latestUpdates = newLatestUpdates
+	updatesLock.Unlock()
+
+	log.Tracef("updates: load complete")
+
+	if len(stableUpdates) == 0 {
+		err := loadIndexesFromDisk()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func ScanForLatest(baseDir string, hardFail bool) (latest map[string]string, lastError error) {
+	var added int
+	latest = make(map[string]string)
+
+	filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			lastError = err
+			if hardFail {
+				return err
+			}
+			log.Warningf("updates: could not read %s", path)
+			return nil
+		}
+		if !info.IsDir() {
+			added++
+		}
+
+		relativePath := strings.TrimLeft(strings.TrimPrefix(path, baseDir), "/")
+		identifierPath, version, ok := getIdentifierAndVersion(relativePath)
+		if !ok {
+			return nil
+		}
+
+		// add/update index
+		storedVersion, ok := latest[identifierPath]
+		if ok {
+			// FIXME: this will fail on multi-digit version segments!
+			if version > storedVersion {
+				latest[identifierPath] = version
+			}
+		} else {
+			latest[identifierPath] = version
+		}
+
+		return nil
+	})
+
+	if lastError != nil {
+		if hardFail {
+			return nil, lastError
+		}
+		if added == 0 {
+			return latest, lastError
+		}
+	}
+	return latest, nil
+}
+
+func loadIndexesFromDisk() error {
+	data, err := ioutil.ReadFile(filepath.Join(updateStoragePath, "stable.json"))
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Infof("updates: stable.json does not yet exist, waiting for first update cycle")
+			return nil
+		}
+		return err
+	}
+
+	newStableUpdates := make(map[string]string)
+	err = json.Unmarshal(data, &newStableUpdates)
+	if err != nil {
+		return err
+	}
+
+	if len(newStableUpdates) == 0 {
+		return errors.New("stable.json is empty")
+	}
+
+	log.Tracef("updates: loaded stable.json")
+
+	updatesLock.Lock()
+	stableUpdates = newStableUpdates
+	updatesLock.Unlock()
+
+	return nil
+}

updates/latest_test.go

@@ -0,0 +1,73 @@
+package updates
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func testLoadLatestScope(t *testing.T, basePath, filePath, expectedIdentifier, expectedVersion string) {
+	fullPath := filepath.Join(basePath, filePath)
+
+	// create dir
+	dirPath := filepath.Dir(fullPath)
+	err := os.MkdirAll(dirPath, 0755)
+	if err != nil {
+		t.Fatalf("could not create test dir: %s\n", err)
+		return
+	}
+
+	// touch file
+	err = ioutil.WriteFile(fullPath, []byte{}, 0644)
+	if err != nil {
+		t.Fatalf("could not create test file: %s\n", err)
+		return
+	}
+
+	// run loadLatestScope
+	latest, err := ScanForLatest(basePath, true)
+	if err != nil {
+		t.Errorf("could not update latest: %s\n", err)
+		return
+	}
+	for key, val := range latest {
+		latestUpdates[key] = val
+	}
+
+	// test result
+	version, ok := latestUpdates[expectedIdentifier]
+	if !ok {
+		t.Errorf("identifier %s not in map", expectedIdentifier)
+		t.Errorf("current map: %v", latestUpdates)
+	}
+	if version != expectedVersion {
+		t.Errorf("unexpected version for %s: %s", filePath, version)
+	}
+}
+
+func TestLoadLatestScope(t *testing.T) {
+
+	updatesLock.Lock()
+	defer updatesLock.Unlock()
+
+	tmpDir, err := ioutil.TempDir("", "testing_")
+	if err != nil {
+		t.Fatalf("could not create test dir: %s\n", err)
+		return
+	}
+	defer os.RemoveAll(tmpDir)
+
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "1.2.3")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3b.zip", "all/ui/assets.zip", "1.2.3b")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "1.2.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "1.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v2-3-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-3.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-2-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "all/ui/assets_v1-3-4.zip", "all/ui/assets.zip", "2.3.4")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "1.2.3")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v2-1-1", "os_platform/portmaster/portmaster", "2.1.1")
+	testLoadLatestScope(t, tmpDir, "os_platform/portmaster/portmaster_v1-2-3", "os_platform/portmaster/portmaster", "2.1.1")
+
+}

updates/main.go

@@ -0,0 +1,95 @@
+package updates
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+
+	"github.com/Safing/portbase/database"
+	"github.com/Safing/portbase/modules"
+)
+
+var (
+	updateStoragePath string
+)
+
+func init() {
+	modules.Register("updates", prep, start, nil, "database")
+}
+
+func prep() error {
+	updateStoragePath = filepath.Join(database.GetDatabaseRoot(), "updates")
+
+	err := checkUpdateDirs()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func start() error {
+	err := ReloadLatest()
+	if err != nil {
+		return err
+	}
+
+	go updater()
+	return nil
+}
+
+func stop() error {
+	return os.RemoveAll(filepath.Join(updateStoragePath, "tmp"))
+}
+
+func checkUpdateDirs() error {
+	// all
+	err := checkDir(filepath.Join(updateStoragePath, "all"))
+	if err != nil {
+		return err
+	}
+
+	// os_platform
+	err = checkDir(filepath.Join(updateStoragePath, fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)))
+	if err != nil {
+		return err
+	}
+
+	// tmp
+	err = checkDir(filepath.Join(updateStoragePath, "tmp"))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func checkDir(dirPath string) error {
+	f, err := os.Stat(dirPath)
+	if err == nil {
+		// file exists
+		if f.IsDir() {
+			return nil
+		}
+		err = os.Remove(dirPath)
+		if err != nil {
+			return fmt.Errorf("could not remove file %s to place dir: %s", dirPath, err)
+		}
+		err = os.MkdirAll(dirPath, 0755)
+		if err != nil {
+			return fmt.Errorf("could not create dir %s: %s", dirPath, err)
+		}
+		return nil
+	}
+	// file does not exist
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(dirPath, 0755)
+		if err != nil {
+			return fmt.Errorf("could not create dir %s: %s", dirPath, err)
+		}
+		return nil
+	}
+	// other error
+	return fmt.Errorf("failed to access %s: %s", dirPath, err)
+}

updates/updater.go

@@ -0,0 +1,88 @@
+package updates
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"path/filepath"
+	"time"
+
+	"github.com/Safing/portbase/log"
+)
+
+func updater() {
+	time.Sleep(10 * time.Second)
+	for {
+		err := checkForUpdates()
+		if err != nil {
+			log.Warningf("updates: failed to check for updates: %s", err)
+		}
+		time.Sleep(1 * time.Hour)
+	}
+}
+
+func checkForUpdates() error {
+
+	// download new index
+	var data []byte
+	var err error
+	for tries := 0; tries < 3; tries++ {
+		data, err = fetchData("stable.json", tries)
+		if err == nil {
+			break
+		}
+	}
+	if err != nil {
+		return err
+	}
+
+	newStableUpdates := make(map[string]string)
+	err = json.Unmarshal(data, &newStableUpdates)
+	if err != nil {
+		return err
+	}
+
+	if len(newStableUpdates) == 0 {
+		return errors.New("stable.json is empty")
+	}
+
+	// FIXINSTABLE: correct log line
+	log.Infof("updates: downloaded new update index: stable.json (alpha until we actually reach stable)")
+
+	// update existing files
+	log.Tracef("updates: updating existing files")
+	updatesLock.RLock()
+	for identifier, newVersion := range newStableUpdates {
+		oldVersion, ok := latestUpdates[identifier]
+		if ok && newVersion != oldVersion {
+
+			filePath := getVersionedPath(identifier, newVersion)
+			realFilePath := filepath.Join(updateStoragePath, filePath)
+			for tries := 0; tries < 3; tries++ {
+				err := fetchFile(realFilePath, filePath, tries)
+				if err == nil {
+					break
+				}
+			}
+			if err != nil {
+				log.Warningf("failed to update %s to %s: %s", identifier, newVersion, err)
+			}
+
+		}
+	}
+	updatesLock.RUnlock()
+	log.Tracef("updates: finished updating existing files")
+
+	// update stable index
+	updatesLock.Lock()
+	stableUpdates = newStableUpdates
+	updatesLock.Unlock()
+
+	// save stable index
+	err = ioutil.WriteFile(filepath.Join(updateStoragePath, "stable.json"), data, 0644)
+	if err != nil {
+		log.Warningf("updates: failed to save new version of stable.json: %s", err)
+	}
+
+	return nil
+}

updates/uptool/.gitignore

@@ -0,0 +1 @@
+uptool

updates/uptool/root.go

@@ -0,0 +1,23 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/spf13/cobra"
+)
+
+var rootCmd = &cobra.Command{
+	Use:   "uptool",
+	Short: "helper tool for the update process",
+	Run: func(cmd *cobra.Command, args []string) {
+		cmd.Usage()
+	},
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

updates/uptool/scan.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/Safing/portmaster/updates"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(scanCmd)
+}
+
+var scanCmd = &cobra.Command{
+	Use:   "scan",
+	Short: "Scan the current directory and print the result",
+	RunE:  scan,
+}
+
+func scan(cmd *cobra.Command, args []string) error {
+
+	latest, err := updates.ScanForLatest(".", true)
+	if err != nil {
+		return err
+	}
+
+	data, err := json.MarshalIndent(latest, "", " ")
+	if err != nil {
+		return err
+	}
+
+	fmt.Println(string(data))
+	return nil
+}