Adapt modules to new core module and dir structure handling

firewall/api.go

@@ -9,7 +9,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/safing/portbase/database"
+	"github.com/safing/portbase/utils"
+	"github.com/safing/portmaster/core/structure"
+
 	"github.com/safing/portbase/log"
 
 	"github.com/safing/portmaster/network/packet"
@@ -19,7 +21,7 @@ import (
 )
 
 var (
-	dbRoot string
+	dataRoot *utils.DirStructure
 
 	apiAddressSet bool
 	apiIP         net.IP
@@ -27,7 +29,7 @@ var (
 )
 
 func prepAPIAuth() error {
-	dbRoot = database.GetDatabaseRoot()
+	dataRoot = structure.Root()
 	return api.SetAuthenticator(apiAuthenticator)
 }
 
@@ -41,6 +43,10 @@ func startAPIAuth() {
 }
 
 func apiAuthenticator(s *http.Server, r *http.Request) (grantAccess bool, err error) {
+	if devMode() {
+		return true, nil
+	}
+
 	// get local IP/Port
 	localIP, localPort, err := parseHostPort(s.Addr)
 	if err != nil {
@@ -64,7 +70,7 @@ func apiAuthenticator(s *http.Server, r *http.Request) (grantAccess bool, err er
 	// go up up to two levels, if we don't match
 	for i := 0; i < 3; i++ {
 		// check if the requesting process is in database root / updates dir
-		if strings.HasPrefix(proc.Path, dbRoot) {
+		if strings.HasPrefix(proc.Path, dataRoot.Path) {
 			return true, nil
 		}
 		// add checked process to list
@@ -79,8 +85,8 @@ func apiAuthenticator(s *http.Server, r *http.Request) (grantAccess bool, err er
 		}
 	}
 
-	log.Debugf("firewall: denying api access to %s - also checked %s (trusted root is %s)", procsChecked[0], strings.Join(procsChecked[1:], " "), dbRoot)
+	log.Debugf("firewall: denying api access to %s - also checked %s (trusted root is %s)", procsChecked[0], strings.Join(procsChecked[1:], " "), dataRoot.Path)
-	return true, nil
+	return false, nil
 }
 
 func parseHostPort(address string) (net.IP, uint16, error) {

firewall/config.go

@@ -9,6 +9,7 @@ var (
 	permanentVerdicts  config.BoolOption
 	filterDNSByScope   status.SecurityLevelOption
 	filterDNSByProfile status.SecurityLevelOption
+	devMode            config.BoolOption
 )
 
 func registerConfig() error {
@@ -55,5 +56,7 @@ func registerConfig() error {
 	}
 	filterDNSByProfile = status.ConfigIsActiveConcurrent("firewall/filterDNSByProfile")
 
+	devMode = config.Concurrent.GetAsBool("firewall/permanentVerdicts", true)
+
 	return nil
 }

nameserver/takeover.go

@@ -26,7 +26,7 @@ func checkForConflictingService(err error) {
 	(&notifications.Notification{
 		ID:      "nameserver-stopped-conflicting-service",
 		Message: fmt.Sprintf("Portmaster stopped a conflicting name service (pid %d) to gain required system integration.", pid),
-	}).Init().Save()
+	}).Save()
 
 	// wait for a short duration for the other service to shut down
 	time.Sleep(100 * time.Millisecond)

network/module.go

@@ -5,7 +5,7 @@ import (
 )
 
 func init() {
-	modules.Register("network", nil, start, nil, "database")
+	modules.Register("network", nil, start, nil, "core")
 }
 
 func start() error {

pmctl/lock.go

@@ -13,7 +13,7 @@ import (
 )
 
 func checkAndCreateInstanceLock(name string) (pid int32, err error) {
-	lockFilePath := filepath.Join(databaseRootDir, fmt.Sprintf("%s-lock.pid", name))
+	lockFilePath := filepath.Join(dataRoot.Path, fmt.Sprintf("%s-lock.pid", name))
 
 	// read current pid file
 	data, err := ioutil.ReadFile(lockFilePath)
@@ -49,10 +49,10 @@ func checkAndCreateInstanceLock(name string) (pid int32, err error) {
 }
 
 func createInstanceLock(lockFilePath string) error {
-	// create database dir
+	// check data root dir
-	err := os.MkdirAll(databaseRootDir, 0777)
+	err := dataRoot.Ensure()
 	if err != nil {
-		log.Printf("failed to create base folder: %s\n", err)
+		log.Printf("failed to check data root dir: %s\n", err)
 	}
 
 	// create lock file
@@ -65,6 +65,6 @@ func createInstanceLock(lockFilePath string) error {
 }
 
 func deleteInstanceLock(name string) error {
-	lockFilePath := filepath.Join(databaseRootDir, fmt.Sprintf("%s-lock.pid", name))
+	lockFilePath := filepath.Join(dataRoot.Path, fmt.Sprintf("%s-lock.pid", name))
 	return os.Remove(lockFilePath)
 }

pmctl/logs.go

@@ -18,7 +18,7 @@ import (
 )
 
 func initializeLogFile(logFilePath string, identifier string, updateFile *updates.File) *os.File {
-	logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE, 0644)
+	logFile, err := os.OpenFile(logFilePath, os.O_RDWR|os.O_CREATE, 0444)
 	if err != nil {
 		log.Printf("failed to create log file %s: %s\n", logFilePath, err)
 		return nil
@@ -74,12 +74,11 @@ func finalizeLogFile(logFile *os.File, logFilePath string) {
 }
 
 func initControlLogFile() *os.File {
-	// create logging dir
+	// check logging dir
-	logFileBasePath := filepath.Join(databaseRootDir, "logs", "fstree", "control")
+	logFileBasePath := filepath.Join(logsRoot.Path, "fstree", "control")
-	err := os.MkdirAll(logFileBasePath, 0777)
+	err := logsRoot.EnsureAbsPath(logFileBasePath)
 	if err != nil {
-		log.Printf("failed to create log file folder %s: %s\n", logFileBasePath, err)
+		log.Printf("failed to check/create log file folder %s: %s\n", logFileBasePath, err)
-		return nil
 	}
 
 	// open log file
@@ -93,11 +92,11 @@ func logControlError(cErr error) {
 		return
 	}
 
-	// create dir
+	// check logging dir
-	logFileBasePath := filepath.Join(databaseRootDir, "logs", "fstree", "control")
+	logFileBasePath := filepath.Join(logsRoot.Path, "fstree", "control")
-	err := os.MkdirAll(logFileBasePath, 0777)
+	err := logsRoot.EnsureAbsPath(logFileBasePath)
 	if err != nil {
-		log.Printf("failed to create log file folder %s: %s\n", logFileBasePath, err)
+		log.Printf("failed to check/create log file folder %s: %s\n", logFileBasePath, err)
 	}
 
 	// open log file
@@ -113,11 +112,11 @@ func logControlError(cErr error) {
 }
 
 func logControlStack() {
-	// create dir
+	// check logging dir
-	logFileBasePath := filepath.Join(databaseRootDir, "logs", "fstree", "control")
+	logFileBasePath := filepath.Join(logsRoot.Path, "fstree", "control")
-	err := os.MkdirAll(logFileBasePath, 0777)
+	err := logsRoot.EnsureAbsPath(logFileBasePath)
 	if err != nil {
-		log.Printf("failed to create log file folder %s: %s\n", logFileBasePath, err)
+		log.Printf("failed to check/create log file folder %s: %s\n", logFileBasePath, err)
 	}
 
 	// open log file

pmctl/main.go

@@ -10,14 +10,22 @@ import (
 	"strings"
 	"syscall"
 
+	"github.com/safing/portmaster/core/structure"
+	"github.com/safing/portmaster/updates"
+
+	"github.com/safing/portbase/utils"
+
 	"github.com/safing/portbase/info"
 	portlog "github.com/safing/portbase/log"
-	"github.com/safing/portmaster/updates"
 	"github.com/spf13/cobra"
 )
 
 var (
-	databaseRootDir  string
+	dataDir     string
+	databaseDir string
+	dataRoot    *utils.DirStructure
+	logsRoot    *utils.DirStructure
+
 	showShortVersion bool
 	showFullVersion  bool
 
@@ -43,7 +51,8 @@ func init() {
 	// Let cobra ignore if we are running as "GUI" or not
 	cobra.MousetrapHelpText = ""
 
-	rootCmd.PersistentFlags().StringVar(&databaseRootDir, "db", "", "set database directory")
+	rootCmd.PersistentFlags().StringVar(&dataDir, "data", "", "set data directory")
+	rootCmd.PersistentFlags().StringVar(&databaseDir, "db", "", "alias to --data (deprecated)")
 	rootCmd.Flags().BoolVar(&showFullVersion, "version", false, "print version")
 	rootCmd.Flags().BoolVar(&showShortVersion, "ver", false, "print version number only")
 }
@@ -123,14 +132,30 @@ func cmdSetup(cmd *cobra.Command, args []string) (err error) {
 	portlog.SetLogLevel(portlog.CriticalLevel)
 
 	if !showShortVersion && !showFullVersion {
-		// set database root
+		// set data root
-		if databaseRootDir != "" {
+		// backwards compatibility
-			// remove redundant escape characters and quotes
+		if dataDir == "" {
-			databaseRootDir = strings.Trim(databaseRootDir, `\"`)
+			dataDir = databaseDir
-			// set updates path
+		}
-			updates.SetDatabaseRoot(databaseRootDir)
+		// check data dir
-		} else {
+		if dataDir == "" {
-			return errors.New("please supply the database directory using the --db flag")
+			return errors.New("please set the data directory using --data=/path/to/data/dir")
+		}
+		// remove redundant escape characters and quotes
+		dataDir = strings.Trim(dataDir, `\"`)
+		// initialize structure
+		err = structure.Initialize(dataDir, 0755)
+		if err != nil {
+			return fmt.Errorf("failed to initialize data root: %s", err)
+		}
+		dataRoot = structure.Root()
+		// manually set updates root (no modules)
+		updates.SetDataRoot(structure.Root())
+		// set up logs root
+		logsRoot = structure.NewRootDir("logs", 0777)
+		err = logsRoot.Ensure()
+		if err != nil {
+			return fmt.Errorf("failed to initialize logs root: %s", err)
 		}
 
 		// warn about CTRL-C on windows

pmctl/run.go

@@ -207,10 +207,10 @@ func execute(opts *Options, args []string) (cont bool, err error) {
 
 	// log files
 	var logFile, errorFile *os.File
-	logFileBasePath := filepath.Join(databaseRootDir, "logs", "fstree", opts.ShortIdentifier)
+	logFileBasePath := filepath.Join(logsRoot.Path, "fstree", opts.ShortIdentifier)
-	err = os.MkdirAll(logFileBasePath, 0777)
+	err = logsRoot.EnsureAbsPath(logFileBasePath)
 	if err != nil {
-		log.Printf("failed to create log file folder %s: %s\n", logFileBasePath, err)
+		log.Printf("failed to check/create log file dir %s: %s\n", logFileBasePath, err)
 	} else {
 		// open log file
 		logFilePath := filepath.Join(logFileBasePath, fmt.Sprintf("%s.log", time.Now().UTC().Format("2006-02-01-15-04-05")))

updates/fetch.go

@@ -15,7 +15,6 @@ import (
 	"github.com/google/renameio"
 
 	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/utils"
 )
 
 var (
@@ -38,13 +37,13 @@ func fetchFile(realFilepath, updateFilepath string, tries int) error {
 
 	// check destination dir
 	dirPath := filepath.Dir(realFilepath)
-	err = utils.EnsureDirectory(dirPath, 0755)
+	err = updateStorage.EnsureAbsPath(dirPath)
 	if err != nil {
 		return fmt.Errorf("could not create updates folder: %s", dirPath)
 	}
 
 	// open file for writing
-	atomicFile, err := renameio.TempFile(downloadTmpPath, realFilepath)
+	atomicFile, err := renameio.TempFile(tmpStorage.Path, realFilepath)
 	if err != nil {
 		return fmt.Errorf("could not create temp file for download: %s", err)
 	}

updates/get.go

@@ -9,7 +9,6 @@ import (
 	"runtime"
 
 	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/utils"
 )
 
 // Errors
@@ -75,7 +74,7 @@ func loadOrFetchFile(identifier string, fetch bool) (*File, error) {
 	}
 
 	// build final filepath
-	realFilePath := filepath.Join(updateStoragePath, filepath.FromSlash(versionedFilePath))
+	realFilePath := filepath.Join(updateStorage.Path, filepath.FromSlash(versionedFilePath))
 	if _, err := os.Stat(realFilePath); err == nil {
 		// file exists
 		updateUsedStatus(identifier, version)
@@ -83,7 +82,7 @@ func loadOrFetchFile(identifier string, fetch bool) (*File, error) {
 	}
 
 	// check download dir
-	err := utils.EnsureDirectory(downloadTmpPath, 0700)
+	err := tmpStorage.Ensure()
 	if err != nil {
 		return nil, fmt.Errorf("could not prepare tmp directory for download: %s", err)
 	}

updates/latest.go

@@ -26,14 +26,14 @@ func LoadLatest() error {
 
 	// all
 	prefix := "all"
-	new, err1 := ScanForLatest(filepath.Join(updateStoragePath, prefix), false)
+	new, err1 := ScanForLatest(filepath.Join(updateStorage.Path, prefix), false)
 	for key, val := range new {
 		newLocalUpdates[filepath.ToSlash(filepath.Join(prefix, key))] = val
 	}
 
 	// os_platform
 	prefix = fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH)
-	new, err2 := ScanForLatest(filepath.Join(updateStoragePath, prefix), false)
+	new, err2 := ScanForLatest(filepath.Join(updateStorage.Path, prefix), false)
 	for key, val := range new {
 		newLocalUpdates[filepath.ToSlash(filepath.Join(prefix, key))] = val
 	}
@@ -120,7 +120,7 @@ func ScanForLatest(baseDir string, hardFail bool) (latest map[string]string, las
 
 // LoadIndexes loads the current update indexes from disk.
 func LoadIndexes() error {
-	data, err := ioutil.ReadFile(filepath.Join(updateStoragePath, "stable.json"))
+	data, err := ioutil.ReadFile(filepath.Join(updateStorage.Path, "stable.json"))
 	if err != nil {
 		return err
 	}

updates/main.go

@@ -3,27 +3,30 @@ package updates
 import (
 	"errors"
 	"os"
-	"path/filepath"
 	"runtime"
 
-	"github.com/safing/portbase/database"
+	"github.com/safing/portmaster/core/structure"
+
 	"github.com/safing/portbase/info"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
 	"github.com/safing/portbase/utils"
 )
 
-var (
+const (
-	updateStoragePath string
+	isWindows = runtime.GOOS == "windows"
-	downloadTmpPath   string
-	isWindows         = runtime.GOOS == "windows"
 )
 
-// SetDatabaseRoot tells the updates module where the database is - and where to put its stuff.
+var (
-func SetDatabaseRoot(path string) {
+	updateStorage *utils.DirStructure
-	if updateStoragePath == "" {
+	tmpStorage    *utils.DirStructure
-		updateStoragePath = filepath.Join(path, "updates")
+)
-		downloadTmpPath = filepath.Join(updateStoragePath, "tmp")
+
+// SetDataRoot sets the data root from which the updates module derives its paths.
+func SetDataRoot(root *utils.DirStructure) {
+	if root != nil && updateStorage == nil {
+		updateStorage = root.ChildDir("updates", 0755)
+		tmpStorage = updateStorage.ChildDir("tmp", 0777)
 	}
 }
 
@@ -32,19 +35,12 @@ func init() {
 }
 
 func prep() error {
-	dbRoot := database.GetDatabaseRoot()
+	SetDataRoot(structure.Root())
-	if dbRoot == "" {
+	if updateStorage == nil {
-		return errors.New("database root is not set")
+		return errors.New("update storage path is not set")
-	}
-	updateStoragePath = filepath.Join(dbRoot, "updates")
-	downloadTmpPath = filepath.Join(updateStoragePath, "tmp")
-
-	err := utils.EnsureDirectory(updateStoragePath, 0755)
-	if err != nil {
-		return err
 	}
 
-	err = utils.EnsureDirectory(downloadTmpPath, 0700)
+	err := updateStorage.Ensure()
 	if err != nil {
 		return err
 	}
@@ -87,5 +83,5 @@ func start() error {
 
 func stop() error {
 	// delete download tmp dir
-	return os.RemoveAll(downloadTmpPath)
+	return os.RemoveAll(tmpStorage.Path)
 }

updates/notify.go

@@ -33,7 +33,7 @@ func updateNotifier() {
 					Message: fmt.Sprintf("There is an update available for the Portmaster core (v%s), please restart the Portmaster to apply the update.", file.Version()),
 					Type:    notifications.Info,
 					Expires: time.Now().Add(1 * time.Minute).Unix(),
-				}).Init().Save()
+				}).Save()
 
 			}
 		}

updates/updater.go

@@ -12,7 +12,6 @@ import (
 	"time"
 
 	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/utils"
 )
 
 func updater() {
@@ -84,13 +83,13 @@ func UpdateIndexes() (err error) {
 	updatesLock.Unlock()
 
 	// check dir
-	err = utils.EnsureDirectory(updateStoragePath, 0755)
+	err = updateStorage.Ensure()
 	if err != nil {
 		return err
 	}
 
 	// save stable index
-	err = ioutil.WriteFile(filepath.Join(updateStoragePath, "stable.json"), data, 0644)
+	err = ioutil.WriteFile(filepath.Join(updateStorage.Path, "stable.json"), data, 0644)
 	if err != nil {
 		log.Warningf("updates: failed to save new version of stable.json: %s", err)
 	}
@@ -125,6 +124,12 @@ func DownloadUpdates() (err error) {
 	}
 	updatesLock.Unlock()
 
+	// check download dir
+	err = tmpStorage.Ensure()
+	if err != nil {
+		return fmt.Errorf("could not prepare tmp directory for download: %s", err)
+	}
+
 	// RLock for the remaining function
 	updatesLock.RLock()
 	defer updatesLock.RUnlock()
@@ -137,7 +142,7 @@ func DownloadUpdates() (err error) {
 
 			log.Tracef("updates: updating %s to %s", identifier, newVersion)
 			filePath := GetVersionedPath(identifier, newVersion)
-			realFilePath := filepath.Join(updateStoragePath, filePath)
+			realFilePath := filepath.Join(updateStorage.Path, filePath)
 			for tries := 0; tries < 3; tries++ {
 				err = fetchFile(realFilePath, filePath, tries)
 				if err == nil {
@@ -153,9 +158,9 @@ func DownloadUpdates() (err error) {
 	log.Tracef("updates: finished updating existing files")
 
 	// remove tmp folder after we are finished
-	err = os.RemoveAll(downloadTmpPath)
+	err = os.RemoveAll(tmpStorage.Path)
 	if err != nil {
-		log.Tracef("updates: failed to remove tmp dir %s after downloading updates: %s", updateStoragePath, err)
+		log.Tracef("updates: failed to remove tmp dir %s after downloading updates: %s", updateStorage.Path, err)
 	}
 
 	return nil

updates/upgrader.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/google/renameio"
-	"github.com/safing/portbase/utils"
 
 	"github.com/safing/portbase/log"
 
@@ -35,7 +34,7 @@ func runFileUpgrades() error {
 	}
 
 	// update portmaster-control in data root
-	rootControlPath := filepath.Join(filepath.Dir(updateStoragePath), filename)
+	rootControlPath := filepath.Join(filepath.Dir(updateStorage.Path), filename)
 	err = upgradeFile(rootControlPath, newFile)
 	if err != nil {
 		return err
@@ -76,6 +75,12 @@ func upgradeFile(fileToUpgrade string, file *File) error {
 		fileExists = true
 	}
 
+	// ensure that the tmp dir exists
+	err = tmpStorage.Ensure()
+	if err != nil {
+		return fmt.Errorf("unable to create directory for upgrade process: %s", err)
+	}
+
 	if fileExists {
 		// get current version
 		var currentVersion string
@@ -103,14 +108,8 @@ func upgradeFile(fileToUpgrade string, file *File) error {
 		err = os.Remove(fileToUpgrade)
 		if err != nil {
 			// maybe we're on windows and it's in use, try moving
-			// create dir
-			err = utils.EnsureDirectory(downloadTmpPath, 0700)
-			if err != nil {
-				return fmt.Errorf("unable to create directory for upgrade process: %s", err)
-			}
-			// move
 			err = os.Rename(fileToUpgrade, filepath.Join(
-				downloadTmpPath,
+				tmpStorage.Path,
 				fmt.Sprintf(
 					"%s-%d%s",
 					GetVersionedPath(filepath.Base(fileToUpgrade), currentVersion),
@@ -154,7 +153,7 @@ func upgradeFile(fileToUpgrade string, file *File) error {
 
 func copyFile(srcPath, dstPath string) (err error) {
 	// open file for writing
-	atomicDstFile, err := renameio.TempFile(downloadTmpPath, dstPath)
+	atomicDstFile, err := renameio.TempFile(tmpStorage.Path, dstPath)
 	if err != nil {
 		return fmt.Errorf("could not create temp file for atomic copy: %s", err)
 	}
@@ -183,5 +182,5 @@ func copyFile(srcPath, dstPath string) (err error) {
 }
 
 func cleanOldUpgradedFiles() error {
-	return os.RemoveAll(downloadTmpPath)
+	return os.RemoveAll(tmpStorage.Path)
 }