Add support for staging and purging

cmds/portmaster-start/main.go

@@ -22,6 +22,7 @@ import (
 
 var (
 	dataDir    string
+	staging    bool
 	maxRetries int
 	dataRoot   *utils.DirStructure
 	logsRoot   *utils.DirStructure
@@ -41,8 +42,8 @@ var (
 		Use:   "portmaster-start",
 		Short: "Start Portmaster components",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
-			mustLoadIndex := cmd == updatesCmd
+			mustLoadIndex := indexRequired(cmd)
-			if err := configureDataRoot(mustLoadIndex); err != nil {
+			if err := configureRegistry(mustLoadIndex); err != nil {
 				return err
 			}
 
@@ -64,8 +65,9 @@ func init() {
 	{
 		flags.StringVar(&dataDir, "data", "", "Configures the data directory. Alternatively, this can also be set via the environment variable PORTMASTER_DATA.")
 		flags.StringVar(&registry.UserAgent, "update-agent", "Start", "Sets the user agent for requests to the update server")
+		flags.BoolVar(&staging, "staging", false, "Use staging update channel (for testing only)")
 		flags.IntVar(&maxRetries, "max-retries", 5, "Maximum number of retries when starting a Portmaster component")
-		flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdid.")
+		flags.BoolVar(&stdinSignals, "input-signals", false, "Emulate signals using stdin.")
 		_ = rootCmd.MarkPersistentFlagDirname("data")
 		_ = flags.MarkHidden("input-signals")
 	}
@@ -131,34 +133,32 @@ func initCobra() {
 	portlog.SetLogLevel(portlog.CriticalLevel)
 }
 
-func configureDataRoot(mustLoadIndex bool) error {
+func configureRegistry(mustLoadIndex bool) error {
-	// The data directory is not
+	// If dataDir is not set, check the environment variable.
-	// check for environment variable
-	// PORTMASTER_DATA
 	if dataDir == "" {
 		dataDir = os.Getenv("PORTMASTER_DATA")
 	}
 
-	// if it's still empty try to auto-detect it
+	// If it's still empty, try to auto-detect it.
 	if dataDir == "" {
 		dataDir = detectInstallationDir()
 	}
 
-	// finally, if it's still empty the user must provide it
+	// Finally, if it's still empty, the user must provide it.
 	if dataDir == "" {
 		return errors.New("please set the data directory using --data=/path/to/data/dir")
 	}
 
-	// remove redundant escape characters and quotes
+	// Remove left over quotes.
 	dataDir = strings.Trim(dataDir, `\"`)
-	// initialize dataroot
+	// Initialize data root.
 	err := dataroot.Initialize(dataDir, 0755)
 	if err != nil {
 		return fmt.Errorf("failed to initialize data root: %s", err)
 	}
 	dataRoot = dataroot.Root()
 
-	// initialize registry
+	// Initialize registry.
 	err = registry.Initialize(dataRoot.ChildDir("updates", 0755))
 	if err != nil {
 		return err
@@ -177,6 +177,19 @@ func configureDataRoot(mustLoadIndex bool) error {
 	// Beta:   true,
 	// })
 
+	if stagingActive() {
+		// Set flag no matter how staging was activated.
+		staging = true
+
+		log.Println("WARNING: staging environment is active.")
+
+		registry.AddIndex(updater.Index{
+			Path:   "staging.json",
+			Stable: true,
+			Beta:   true,
+		})
+	}
+
 	return updateRegistryIndex(mustLoadIndex)
 }
 
@@ -233,3 +246,14 @@ func detectInstallationDir() string {
 
 	return parent
 }
+
+func stagingActive() bool {
+	// Check flag and env variable.
+	if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" {
+		return true
+	}
+
+	// Check if staging index is present and acessible.
+	_, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json"))
+	return err == nil
+}

cmds/portmaster-start/update.go

@@ -3,22 +3,49 @@ package main
 import (
 	"context"
 	"fmt"
+	"os"
 	"runtime"
 
 	"github.com/safing/portbase/log"
 	"github.com/spf13/cobra"
 )
 
+var reset bool
+
 func init() {
-	rootCmd.AddCommand(updatesCmd)
+	rootCmd.AddCommand(updateCmd)
+	rootCmd.AddCommand(purgeCmd)
+
+	flags := updateCmd.Flags()
+	flags.BoolVar(&reset, "reset", false, "Delete all resources and re-download the basic set")
 }
 
-var updatesCmd = &cobra.Command{
+var (
-	Use:   "update",
+	updateCmd = &cobra.Command{
-	Short: "Run a manual update process",
+		Use:   "update",
-	RunE: func(cmd *cobra.Command, args []string) error {
+		Short: "Run a manual update process",
-		return downloadUpdates()
+		RunE: func(cmd *cobra.Command, args []string) error {
-	},
+			return downloadUpdates()
+		},
+	}
+
+	purgeCmd = &cobra.Command{
+		Use:   "purge",
+		Short: "Remove old resource versions that are superseded by at least three versions",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return purge()
+		},
+	}
+)
+
+func indexRequired(cmd *cobra.Command) bool {
+	switch cmd {
+	case updateCmd,
+		purgeCmd:
+		return true
+	default:
+		return false
+	}
 }
 
 func downloadUpdates() error {
@@ -26,8 +53,9 @@ func downloadUpdates() error {
 	if onWindows {
 		registry.MandatoryUpdates = []string{
 			platform("core/portmaster-core.exe"),
+			platform("kext/portmaster-kext.dll"),
+			platform("kext/portmaster-kext.sys"),
 			platform("start/portmaster-start.exe"),
-			platform("app/portmaster-app.exe"),
 			platform("notifier/portmaster-notifier.exe"),
 			platform("notifier/portmaster-snoretoast.exe"),
 		}
@@ -35,7 +63,6 @@ func downloadUpdates() error {
 		registry.MandatoryUpdates = []string{
 			platform("core/portmaster-core"),
 			platform("start/portmaster-start"),
-			platform("app/portmaster-app"),
 			platform("notifier/portmaster-notifier"),
 		}
 	}
@@ -43,10 +70,64 @@ func downloadUpdates() error {
 	// add updates that we require on all platforms.
 	registry.MandatoryUpdates = append(
 		registry.MandatoryUpdates,
-		"all/ui/modules/base.zip",
+		platform("app/portmaster-app.zip"),
+		"all/ui/modules/portmaster.zip",
 	)
 
-	log.SetLogLevel(log.InfoLevel)
+	// Add assets that need unpacking.
+	registry.AutoUnpack = []string{
+		platform("app/portmaster-app.zip"),
+	}
+
+	// logging is configured as a persistent pre-run method inherited from
+	// the root command but since we don't use run.Run() we need to start
+	// logging ourself.
+	log.SetLogLevel(log.TraceLevel)
+	err := log.Start()
+	if err != nil {
+		fmt.Printf("failed to start logging: %s\n", err)
+	}
+	defer log.Shutdown()
+
+	if reset {
+		// Delete storage.
+		err = os.RemoveAll(registry.StorageDir().Path)
+		if err != nil {
+			return fmt.Errorf("failed to reset update dir: %s", err)
+		}
+		err = registry.StorageDir().Ensure()
+		if err != nil {
+			return fmt.Errorf("failed to create update dir: %s", err)
+		}
+
+		// Reset registry state.
+		registry.Reset()
+	}
+
+	// Update all indexes.
+	err = registry.UpdateIndexes(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	// Download all required updates.
+	err = registry.DownloadUpdates(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	// Select versions and unpack the selected.
+	registry.SelectVersions()
+	err = registry.UnpackResources()
+	if err != nil {
+		return fmt.Errorf("failed to unpack resources: %s", err)
+	}
+
+	return nil
+}
+
+func purge() error {
+	log.SetLogLevel(log.TraceLevel)
 
 	// logging is configured as a persistent pre-run method inherited from
 	// the root command but since we don't use run.Run() we need to start
@@ -57,7 +138,8 @@ func downloadUpdates() error {
 	}
 	defer log.Shutdown()
 
-	return registry.DownloadUpdates(context.TODO())
+	registry.Purge(3)
+	return nil
 }
 
 func platform(identifier string) string {

cmds/portmaster-start/version.go

@@ -20,9 +20,9 @@ var versionCmd = &cobra.Command{
 	Args:  cobra.NoArgs,
 	PersistentPreRunE: func(*cobra.Command, []string) error {
 		if showAllVersions {
-			// if we are going to show all component versions
+			// If we are going to show all component versions,
-			// we need the dataroot to be configured.
+			// we need the registry to be configured.
-			if err := configureDataRoot(false); err != nil {
+			if err := configureRegistry(false); err != nil {
 				return err
 			}
 		}

cmds/updatemgr/.gitignore

@@ -0,0 +1,2 @@
+updatemgr
+updatemgr.exe

cmds/updatemgr/confirm.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"strings"
+)
+
+func confirm(msg string) bool {
+	fmt.Printf("%s: [y|n] ", msg)
+
+	scanner := bufio.NewScanner(os.Stdin)
+	ok := scanner.Scan()
+	if ok && strings.TrimSpace(scanner.Text()) == "y" {
+		return true
+	}
+
+	return false
+}

cmds/updatemgr/main.go

@@ -12,7 +12,7 @@ import (
 var registry *updater.ResourceRegistry
 
 var rootCmd = &cobra.Command{
-	Use:   "uptool",
+	Use:   "updatemgr",
 	Short: "A simple tool to assist in the update and release process",
 	Args:  cobra.ExactArgs(1),
 	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {

cmds/updatemgr/purge.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portbase/updater"
+)
+
+func init() {
+	rootCmd.AddCommand(purgeCmd)
+}
+
+var purgeCmd = &cobra.Command{
+	Use:   "purge",
+	Short: "Remove old resource versions that are superseded by at least three versions",
+	Args:  cobra.ExactArgs(1),
+	RunE:  purge,
+}
+
+func purge(cmd *cobra.Command, args []string) error {
+	log.SetLogLevel(log.TraceLevel)
+	err := log.Start()
+	if err != nil {
+		fmt.Printf("failed to start logging: %s\n", err)
+	}
+	defer log.Shutdown()
+
+	registry.AddIndex(updater.Index{
+		Path:   "stable.json",
+		Stable: true,
+		Beta:   false,
+	})
+
+	registry.AddIndex(updater.Index{
+		Path:   "beta.json",
+		Stable: false,
+		Beta:   true,
+	})
+
+	err = registry.LoadIndexes(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	err = scanStorage()
+	if err != nil {
+		return err
+	}
+
+	registry.SelectVersions()
+	registry.Purge(3)
+
+	return nil
+}

cmds/updatemgr/staging.go

@@ -0,0 +1,122 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/safing/portbase/updater"
+	"github.com/spf13/cobra"
+)
+
+var (
+	stageReset bool
+)
+
+func init() {
+	rootCmd.AddCommand(stageCmd)
+	stageCmd.Flags().BoolVar(&stageReset, "reset", false, "Reset staging assets")
+}
+
+var stageCmd = &cobra.Command{
+	Use:   "stage",
+	Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes",
+	Args:  cobra.ExactArgs(1),
+	RunE:  stage,
+}
+
+func stage(cmd *cobra.Command, args []string) error {
+	registry.AddIndex(updater.Index{
+		Path:   "stable.json",
+		Stable: true,
+		Beta:   false,
+	})
+
+	registry.AddIndex(updater.Index{
+		Path:   "beta.json",
+		Stable: false,
+		Beta:   true,
+	})
+
+	err := registry.LoadIndexes(context.TODO())
+	if err != nil {
+		return err
+	}
+
+	err = scanStorage()
+	if err != nil {
+		return err
+	}
+
+	// Check if we want to reset staging instead.
+	if stageReset {
+		for _, stagedPath := range exportStaging(true) {
+			err = os.Remove(stagedPath)
+			if err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	// Export all staged versions and format them.
+	stagingData, err := json.MarshalIndent(exportStaging(false), "", " ")
+	if err != nil {
+		return err
+	}
+
+	// Build destination path.
+	stagingIndexFilePath := filepath.Join(registry.StorageDir().Path, "staging.json")
+
+	// Print preview.
+	fmt.Printf("staging (%s):\n", stagingIndexFilePath)
+	fmt.Println(string(stagingData))
+
+	// Ask for confirmation.
+	if !confirm("\nDo you want to write this index?") {
+		fmt.Println("aborted...")
+		return nil
+	}
+
+	// Write new index to disk.
+	err = ioutil.WriteFile(stagingIndexFilePath, stagingData, 0o644) //nolint:gosec // 0644 is intended
+	if err != nil {
+		return err
+	}
+	fmt.Printf("written %s\n", stagingIndexFilePath)
+
+	return nil
+}
+
+func exportStaging(storagePath bool) map[string]string {
+	// Sort all versions.
+	registry.SetBeta(false)
+	registry.SelectVersions()
+	export := registry.Export()
+
+	// Go through all versions and save the highest version, if not stable or beta.
+	versions := make(map[string]string)
+	for _, rv := range export {
+		// Get highest version.
+		v := rv.Versions[0]
+
+		// Do not take stable or beta releases into account.
+		if v.StableRelease || v.BetaRelease {
+			continue
+		}
+
+		// Add highest version to staging
+		if storagePath {
+			rv.SelectedVersion = v
+			versions[rv.Identifier] = rv.GetFile().Path()
+		} else {
+			versions[rv.Identifier] = v.VersionNumber
+		}
+	}
+
+	return versions
+}

cmds/updatemgr/update.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(updateCmd)
+}
+
+var updateCmd = &cobra.Command{
+	Use:   "update",
+	Short: "Update scans the specified directory and registry the index and symlink structure",
+	Args:  cobra.ExactArgs(1),
+	RunE:  update,
+}
+
+func update(cmd *cobra.Command, args []string) error {
+	err := scanStorage()
+	if err != nil {
+		return err
+	}
+
+	// Export versions.
+	betaData, err := json.MarshalIndent(exportSelected(true), "", " ")
+	if err != nil {
+		return err
+	}
+	stableData, err := json.MarshalIndent(exportSelected(false), "", " ")
+	if err != nil {
+		return err
+	}
+
+	// Build destination paths.
+	betaIndexFilePath := filepath.Join(registry.StorageDir().Path, "beta.json")
+	stableIndexFilePath := filepath.Join(registry.StorageDir().Path, "stable.json")
+
+	// Print previews.
+	fmt.Printf("beta (%s):\n", betaIndexFilePath)
+	fmt.Println(string(betaData))
+	fmt.Printf("\nstable: (%s)\n", stableIndexFilePath)
+	fmt.Println(string(stableData))
+
+	// Ask for confirmation.
+	if !confirm("\nDo you want to write these new indexes (and update latest symlinks)?") {
+		fmt.Println("aborted...")
+		return nil
+	}
+
+	// Write indexes.
+	err = ioutil.WriteFile(betaIndexFilePath, betaData, 0o644) //nolint:gosec // 0644 is intended
+	if err != nil {
+		return err
+	}
+	fmt.Printf("written %s\n", betaIndexFilePath)
+
+	err = ioutil.WriteFile(stableIndexFilePath, stableData, 0o644) //nolint:gosec // 0644 is intended
+	if err != nil {
+		return err
+	}
+	fmt.Printf("written %s\n", stableIndexFilePath)
+
+	// Create symlinks to latest stable versions.
+	symlinksDir := registry.StorageDir().ChildDir("latest", 0o755)
+	err = registry.CreateSymlinks(symlinksDir)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("updated stable symlinks in %s\n", symlinksDir.Path)
+
+	return nil
+}

cmds/uptool/.gitignore

@@ -1 +0,0 @@
-uptool

cmds/uptool/update.go

@@ -1,64 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"path/filepath"
-
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	rootCmd.AddCommand(updateCmd)
-}
-
-var updateCmd = &cobra.Command{
-	Use:   "update",
-	Short: "Update scans the specified directory and registry the index and symlink structure",
-	Args:  cobra.ExactArgs(1),
-	RunE:  update,
-}
-
-func update(cmd *cobra.Command, args []string) error {
-	err := scanStorage()
-	if err != nil {
-		return err
-	}
-
-	// export beta
-	data, err := json.MarshalIndent(exportSelected(true), "", " ")
-	if err != nil {
-		return err
-	}
-	// print
-	fmt.Println("beta:")
-	fmt.Println(string(data))
-	// write index
-	err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "beta.json"), data, 0o644) //nolint:gosec // 0644 is intended
-	if err != nil {
-		return err
-	}
-
-	// export stable
-	data, err = json.MarshalIndent(exportSelected(false), "", " ")
-	if err != nil {
-		return err
-	}
-	// print
-	fmt.Println("\nstable:")
-	fmt.Println(string(data))
-	// write index
-	err = ioutil.WriteFile(filepath.Join(registry.StorageDir().Dir, "stable.json"), data, 0o644) //nolint:gosec // 0644 is intended
-	if err != nil {
-		return err
-	}
-	// create symlinks
-	err = registry.CreateSymlinks(registry.StorageDir().ChildDir("latest", 0o755))
-	if err != nil {
-		return err
-	}
-	fmt.Println("\nstable symlinks created")
-
-	return nil
-}

updates/export.go

@@ -35,6 +35,8 @@ type versions struct {
 
 	Core      *info.Info
 	Resources map[string]*updater.Resource
+	Beta      bool
+	Staging   bool
 
 	internalSave bool
 }
@@ -43,6 +45,8 @@ func initVersionExport() (err error) {
 	// init export struct
 	versionExport = &versions{
 		internalSave: true,
+		Beta:         registry.Beta,
+		Staging:      staging,
 	}
 	versionExport.SetKey(versionsDBKey)
 

updates/main.go

@@ -51,6 +51,7 @@ var (
 	module            *modules.Module
 	registry          *updater.ResourceRegistry
 	userAgentFromFlag string
+	staging           bool
 
 	updateTask          *modules.Task
 	updateASAP          bool
@@ -78,6 +79,7 @@ func init() {
 	module.RegisterEvent(ResourceUpdateEvent)
 
 	flag.StringVar(&userAgentFromFlag, "update-agent", "", "Sets the user agent for requests to the update server")
+	flag.BoolVar(&staging, "staging", false, "Use staging update channel (for testing only)")
 
 	// initialize mandatory updates
 	if onWindows {
@@ -182,6 +184,18 @@ func start() error {
 		Beta:   true,
 	})
 
+	if stagingActive() {
+		// Set flag no matter how staging was activated.
+		staging = true
+
+		log.Warning("updates: staging environment is active")
+
+		registry.AddIndex(updater.Index{
+			Path:   "staging.json",
+			Stable: true,
+			Beta:   true,
+		})
+	}
 
 	err = registry.LoadIndexes(module.Ctx)
 	if err != nil {
@@ -308,3 +322,14 @@ func stop() error {
 func platform(identifier string) string {
 	return fmt.Sprintf("%s_%s/%s", runtime.GOOS, runtime.GOARCH, identifier)
 }
+
+func stagingActive() bool {
+	// Check flag and env variable.
+	if staging || os.Getenv("PORTMASTER_STAGING") == "enabled" {
+		return true
+	}
+
+	// Check if staging index is present and acessible.
+	_, err := os.Stat(filepath.Join(registry.StorageDir().Path, "staging.json"))
+	return err == nil
+}