Revamp profile and process handling

Also, introduce the Internal flag to Profiles

network/connection.go

@@ -442,21 +442,6 @@ func (conn *Connection) delete() {
 	dbController.PushUpdate(conn)
 }
 
-// UpdateAndCheck updates profiles and checks whether a reevaluation is needed.
-func (conn *Connection) UpdateAndCheck() (needsReevaluation bool) {
-	p := conn.process.Profile()
-	if p == nil {
-		return false
-	}
-	revCnt := p.Update()
-
-	if conn.profileRevisionCounter != revCnt {
-		conn.profileRevisionCounter = revCnt
-		needsReevaluation = true
-	}
-	return
-}
-
 // SetFirewallHandler sets the firewall handler for this link, and starts a
 // worker to handle the packets.
 func (conn *Connection) SetFirewallHandler(handler FirewallHandler) {

process/database.go

@@ -106,32 +106,33 @@ func CleanProcessStorage(activePIDs map[int]struct{}) {
 
 	// clean primary processes
 	for _, p := range processesCopy {
-		p.Lock()
+		// The PID of a process does not change.
 
-		_, active := activePIDs[p.Pid]
+		// Check if this is a special process.
-		switch {
+		if p.Pid == UnidentifiedProcessID || p.Pid == SystemProcessID {
-		case p.Pid == UnidentifiedProcessID:
+			p.profile.MarkStillActive()
-			// internal
+			continue
-		case p.Pid == SystemProcessID:
-			// internal
-		case active:
-			// process in system process table or recently seen on the network
-		default:
-			// delete now or soon
-			switch {
-			case p.LastSeen == 0:
-				// add last
-				p.LastSeen = time.Now().Unix()
-			case p.LastSeen > threshold:
-				// within keep period
-			default:
-				// delete now
-				log.Tracef("process.clean: deleted %s", p.DatabaseKey())
-				go p.Delete()
-			}
 		}
 
-		p.Unlock()
+		// Check if process is active.
+		_, active := activePIDs[p.Pid]
+		if active {
+			p.profile.MarkStillActive()
+			continue
+		}
+
+		// Process is inactive, start deletion process
+		switch {
+		case p.LastSeen == 0:
+			// add last
+			p.LastSeen = time.Now().Unix()
+		case p.LastSeen > threshold:
+			// within keep period
+		default:
+			// delete now
+			p.Delete()
+			log.Tracef("process: cleaned %s", p.DatabaseKey())
+		}
 	}
 }
 

process/find.go

@@ -30,10 +30,14 @@ func GetProcessByConnection(ctx context.Context, pktInfo *packet.Info) (process
 		return nil, connInbound, err
 	}
 
-	err = process.GetProfile(ctx)
+	changed, err := process.GetProfile(ctx)
 	if err != nil {
 		log.Tracer(ctx).Errorf("process: failed to get profile for process %s: %s", process, err)
 	}
 
+	if changed {
+		process.Save()
+	}
+
 	return process, connInbound, nil
 }

process/process.go

@@ -30,39 +30,35 @@ type Process struct {
 	record.Base
 	sync.Mutex
 
+	// Constant attributes.
+
+	Name      string
 	UserID    int
 	UserName  string
 	UserHome  string
 	Pid       int
 	ParentPid int
 	Path      string
+	ExecName  string
 	Cwd       string
 	CmdLine   string
 	FirstArg  string
 
-	ExecName   string
-	ExecHashes map[string]string
-	// ExecOwner ...
-	// ExecSignature ...
-
 	LocalProfileKey string
 	profile         *profile.LayeredProfile
-	Name            string
+
-	Icon            string
+	// Mutable attributes.
-	// Icon is a path to the icon and is either prefixed "f:" for filepath, "d:" for database cache path or "c:"/"a:" for a the icon key to fetch it from a company / authoritative node and cache it in its own cache.
 
 	FirstSeen int64
 	LastSeen  int64
+	Virtual   bool   // This process is either merged into another process or is not needed.
+	Error     string // Cache errors
 
-	Virtual bool   // This process is either merged into another process or is not needed.
+	ExecHashes map[string]string
-	Error   string // Cache errors
 }
 
 // Profile returns the assigned layered profile.
 func (p *Process) Profile() *profile.LayeredProfile {
-	p.Lock()
-	defer p.Unlock()
-
 	return p.profile
 }
 
@@ -72,8 +68,6 @@ func (p *Process) String() string {
 		return "?"
 	}
 
-	p.Lock()
-	defer p.Unlock()
 	return fmt.Sprintf("%s:%s:%d", p.UserName, p.Path, p.Pid)
 }
 

process/profile.go

@@ -8,35 +8,58 @@ import (
 )
 
 // GetProfile finds and assigns a profile set to the process.
-func (p *Process) GetProfile(ctx context.Context) error {
+func (p *Process) GetProfile(ctx context.Context) (changed bool, err error) {
 	p.Lock()
 	defer p.Unlock()
 
 	// only find profiles if not already done.
 	if p.profile != nil {
 		log.Tracer(ctx).Trace("process: profile already loaded")
-		// mark profile as used
+		// Mark profile as used.
 		p.profile.MarkUsed()
-		return nil
+		return false, nil
 	}
 	log.Tracer(ctx).Trace("process: loading profile")
 
-	// get profile
+	// Check if we need a special profile.
-	localProfile, new, err := profile.FindOrCreateLocalProfileByPath(p.Path)
+	profileID := ""
-	if err != nil {
+	switch p.Pid {
-		return err
+	case UnidentifiedProcessID:
+		profileID = profile.UnidentifiedProfileID
+	case SystemProcessID:
+		profileID = profile.SystemProfileID
 	}
-	// add more information if new
+
+	// Get the (linked) local profile.
+	localProfile, new, err := profile.GetProfile(profile.SourceLocal, profileID, p.Path)
+	if err != nil {
+		return false, err
+	}
+
+	// If the local profile is new, add some information from the process.
 	if new {
 		localProfile.Name = p.ExecName
+
+		// Special profiles will only have a name, but not an ExecName.
+		if localProfile.Name == "" {
+			localProfile.Name = p.Name
+		}
 	}
 
-	// mark profile as used
+	// Mark profile as used.
-	localProfile.MarkUsed()
+	profileChanged := localProfile.MarkUsed()
 
+	// Save the profile if we changed something.
+	if new || profileChanged {
+		err := localProfile.Save()
+		if err != nil {
+			log.Warningf("process: failed to save profile %s: %s", localProfile.ScopedID(), err)
+		}
+	}
+
+	// Assign profile to process.
 	p.LocalProfileKey = localProfile.Key()
-	p.profile = profile.NewLayeredProfile(localProfile)
+	p.profile = localProfile.LayeredProfile()
 
-	go p.Save()
+	return true, nil
-	return nil
 }

process/special.go

@@ -2,10 +2,11 @@ package process
 
 import (
 	"context"
+	"strconv"
 	"time"
 
 	"github.com/safing/portbase/log"
-	"github.com/safing/portmaster/profile"
+	"golang.org/x/sync/singleflight"
 )
 
 // Special Process IDs
@@ -32,53 +33,41 @@ var (
 		ParentPid: SystemProcessID,
 		Name:      "Operating System",
 	}
+
+	getSpecialProcessSingleInflight singleflight.Group
 )
 
 // GetUnidentifiedProcess returns the special process assigned to unidentified processes.
 func GetUnidentifiedProcess(ctx context.Context) *Process {
-	return getSpecialProcess(ctx, UnidentifiedProcessID, unidentifiedProcess, profile.GetUnidentifiedProfile)
+	return getSpecialProcess(ctx, UnidentifiedProcessID, unidentifiedProcess)
 }
 
 // GetSystemProcess returns the special process used for the Kernel.
 func GetSystemProcess(ctx context.Context) *Process {
-	return getSpecialProcess(ctx, SystemProcessID, systemProcess, profile.GetSystemProfile)
+	return getSpecialProcess(ctx, SystemProcessID, systemProcess)
 }
 
-func getSpecialProcess(ctx context.Context, pid int, template *Process, getProfile func() *profile.Profile) *Process {
+func getSpecialProcess(ctx context.Context, pid int, template *Process) *Process {
-	// check storage
+	p, _, _ := getSpecialProcessSingleInflight.Do(strconv.Itoa(pid), func() (interface{}, error) {
-	p, ok := GetProcessFromStorage(pid)
+		// Check if we have already loaded the special process.
-	if ok {
+		process, ok := GetProcessFromStorage(pid)
-		return p
+		if ok {
-	}
+			return process, nil
+		}
 
-	// assign template
+		// Create new process from template
-	p = template
+		process = template
+		process.FirstSeen = time.Now().Unix()
 
-	p.Lock()
+		// Get profile.
-	defer p.Unlock()
+		_, err := process.GetProfile(ctx)
+		if err != nil {
+			log.Tracer(ctx).Errorf("process: failed to get profile for process %s: %s", process, err)
+		}
 
-	if p.FirstSeen == 0 {
+		// Save process to storage.
-		p.FirstSeen = time.Now().Unix()
+		process.Save()
-	}
+		return process, nil
-
+	})
-	// only find profiles if not already done.
+	return p.(*Process)
-	if p.profile != nil {
-		log.Tracer(ctx).Trace("process: special profile already loaded")
-		// mark profile as used
-		p.profile.MarkUsed()
-		return p
-	}
-	log.Tracer(ctx).Trace("process: loading special profile")
-
-	// get profile
-	localProfile := getProfile()
-
-	// mark profile as used
-	localProfile.MarkUsed()
-
-	p.LocalProfileKey = localProfile.Key()
-	p.profile = profile.NewLayeredProfile(localProfile)
-
-	go p.Save()
-	return p
 }

profile/active.go

@@ -7,46 +7,61 @@ import (
 )
 
 const (
-	activeProfileCleanerTickDuration = 10 * time.Minute
+	activeProfileCleanerTickDuration = 1 * time.Minute
-	activeProfileCleanerThreshold    = 1 * time.Hour
+	activeProfileCleanerThreshold    = 5 * time.Minute
 )
 
 var (
-	// TODO: periodically clean up inactive profiles
 	activeProfiles     = make(map[string]*Profile)
 	activeProfilesLock sync.RWMutex
 )
 
 // getActiveProfile returns a cached copy of an active profile and nil if it isn't found.
 func getActiveProfile(scopedID string) *Profile {
-	activeProfilesLock.Lock()
+	activeProfilesLock.RLock()
-	defer activeProfilesLock.Unlock()
+	defer activeProfilesLock.RUnlock()
 
-	profile, ok := activeProfiles[scopedID]
+	activeProfile, ok := activeProfiles[scopedID]
 	if ok {
-		return profile
+		activeProfile.MarkStillActive()
+		return activeProfile
 	}
 
 	return nil
 }
 
-// markProfileActive registers a profile as active.
+// findActiveProfile searched for an active local profile using the linked path.
-func markProfileActive(profile *Profile) {
+func findActiveProfile(linkedPath string) *Profile {
+	activeProfilesLock.RLock()
+	defer activeProfilesLock.RUnlock()
+
+	for _, activeProfile := range activeProfiles {
+		if activeProfile.LinkedPath == linkedPath {
+			activeProfile.MarkStillActive()
+			return activeProfile
+		}
+	}
+
+	return nil
+}
+
+// addActiveProfile registers a active profile.
+func addActiveProfile(profile *Profile) {
 	activeProfilesLock.Lock()
 	defer activeProfilesLock.Unlock()
 
+	profile.MarkStillActive()
 	activeProfiles[profile.ScopedID()] = profile
 }
 
-// markActiveProfileAsOutdated marks an active profile as outdated, so that it will be refetched from the database.
+// markActiveProfileAsOutdated marks an active profile as outdated.
 func markActiveProfileAsOutdated(scopedID string) {
-	activeProfilesLock.Lock()
+	activeProfilesLock.RLock()
-	defer activeProfilesLock.Unlock()
+	defer activeProfilesLock.RUnlock()
 
 	profile, ok := activeProfiles[scopedID]
 	if ok {
 		profile.outdated.Set()
-		delete(activeProfiles, scopedID)
 	}
 }
 
@@ -55,16 +70,12 @@ func cleanActiveProfiles(ctx context.Context) error {
 		select {
 		case <-time.After(activeProfileCleanerTickDuration):
 
-			threshold := time.Now().Add(-activeProfileCleanerThreshold)
+			threshold := time.Now().Add(-activeProfileCleanerThreshold).Unix()
 
 			activeProfilesLock.Lock()
 			for id, profile := range activeProfiles {
-				// get last used
+				// Remove profile if it hasn't been used for a while.
-				profile.Lock()
+				if profile.LastActive() < threshold {
-				lastUsed := profile.lastUsed
-				profile.Unlock()
-				// remove if not used for a while
-				if lastUsed.Before(threshold) {
 					profile.outdated.Set()
 					delete(activeProfiles, id)
 				}

profile/config-update.go

@@ -71,13 +71,9 @@ func updateGlobalConfigProfile(ctx context.Context, data interface{}) error {
 	}
 
 	// build global profile for reference
-	profile := &Profile{
+	profile := New(SourceSpecial, "global-config")
-		ID:           "global-config",
+	profile.Name = "Global Configuration"
-		Source:       SourceSpecial,
+	profile.Internal = true
-		Name:         "Global Configuration",
-		Config:       make(map[string]interface{}),
-		internalSave: true,
-	}
 
 	newConfig := make(map[string]interface{})
 	// fill profile config options

profile/find.go

@@ -1,55 +0,0 @@
-package profile
-
-import (
-	"github.com/safing/portbase/database/query"
-	"github.com/safing/portbase/log"
-)
-
-// FindOrCreateLocalProfileByPath returns an existing or new profile for the given application path.
-func FindOrCreateLocalProfileByPath(fullPath string) (profile *Profile, new bool, err error) {
-	// find local profile
-	it, err := profileDB.Query(
-		query.New(makeProfileKey(SourceLocal, "")).Where(
-			query.Where("LinkedPath", query.SameAs, fullPath),
-		),
-	)
-	if err != nil {
-		return nil, false, err
-	}
-
-	// get first result
-	r := <-it.Next
-	// cancel immediately
-	it.Cancel()
-
-	// return new if none was found
-	if r == nil {
-		profile = New()
-		profile.LinkedPath = fullPath
-		return profile, true, nil
-	}
-
-	// ensure its a profile
-	profile, err = EnsureProfile(r)
-	if err != nil {
-		return nil, false, err
-	}
-
-	// prepare config
-	err = profile.prepConfig()
-	if err != nil {
-		log.Warningf("profiles: profile %s has (partly) invalid configuration: %s", profile.ID, err)
-	}
-
-	// parse config
-	err = profile.parseConfig()
-	if err != nil {
-		log.Warningf("profiles: profile %s has (partly) invalid configuration: %s", profile.ID, err)
-	}
-
-	// mark active
-	markProfileActive(profile)
-
-	// return parsed profile
-	return profile, false, nil
-}

profile/get.go

@@ -0,0 +1,202 @@
+package profile
+
+import (
+	"errors"
+	"os"
+	"strings"
+
+	"github.com/safing/portbase/database"
+
+	"github.com/safing/portbase/dataroot"
+
+	"github.com/safing/portbase/database/query"
+	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/log"
+	"golang.org/x/sync/singleflight"
+)
+
+const (
+	UnidentifiedProfileID = "_unidentified"
+	SystemProfileID       = "_system"
+)
+
+var getProfileSingleInflight singleflight.Group
+
+// GetProfile fetches a profile. This function ensure that the profile loaded
+// is shared among all callers. You must always supply both the scopedID and
+// linkedPath parameters whenever available.
+func GetProfile(source profileSource, id, linkedPath string) (
+	profile *Profile,
+	newProfile bool,
+	err error,
+) {
+	// Select correct key for single in flight.
+	singleInflightKey := linkedPath
+	if singleInflightKey == "" {
+		singleInflightKey = makeScopedID(source, id)
+	}
+
+	p, err, _ := getProfileSingleInflight.Do(singleInflightKey, func() (interface{}, error) {
+		var previousVersion *Profile
+
+		// Fetch profile depending on the available information.
+		switch {
+		case id != "":
+			scopedID := makeScopedID(source, id)
+
+			// Get profile via the scoped ID.
+			// Check if there already is an active and not outdated profile.
+			profile = getActiveProfile(scopedID)
+			if profile != nil {
+				if profile.outdated.IsSet() {
+					previousVersion = profile
+				} else {
+					return profile, nil
+				}
+			}
+			// Get from database.
+			profile, err = getProfile(scopedID)
+
+			// If we cannot find a profile, check if the request is for a special
+			// profile we can create.
+			if errors.Is(err, database.ErrNotFound) {
+				switch id {
+				case UnidentifiedProfileID:
+					profile = New(SourceLocal, UnidentifiedProfileID)
+					newProfile = true
+					err = nil
+				case SystemProfileID:
+					profile = New(SourceLocal, SystemProfileID)
+					newProfile = true
+					err = nil
+				}
+			}
+
+		case linkedPath != "":
+			// Search for profile via a linked path.
+			// Check if there already is an active and not outdated profile for
+			// the linked path.
+			profile = findActiveProfile(linkedPath)
+			if profile != nil {
+				if profile.outdated.IsSet() {
+					previousVersion = profile
+				} else {
+					return profile, nil
+				}
+			}
+			// Get from database.
+			profile, newProfile, err = findProfile(linkedPath)
+
+		default:
+			return nil, errors.New("cannot fetch profile without ID or path")
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		// Process profiles coming directly from the database.
+		// As we don't use any caching, these will be new objects.
+
+		// Mark the profile as being saved internally in order to bypass checks.
+		profile.internalSave = true
+
+		// Add a layeredProfile to local profiles.
+		if profile.Source == SourceLocal {
+			// If we are refetching, assign the layered profile from the previous version.
+			if previousVersion != nil {
+				profile.layeredProfile = previousVersion.layeredProfile
+			}
+
+			// Local profiles must have a layered profile, create a new one if it
+			// does not yet exist.
+			if profile.layeredProfile == nil {
+				profile.layeredProfile = NewLayeredProfile(profile)
+			}
+		}
+
+		// Add the profile to the currently active profiles.
+		addActiveProfile(profile)
+
+		return profile, nil
+	})
+	if err != nil {
+		return nil, false, err
+	}
+	if p == nil {
+		return nil, false, errors.New("profile getter returned nil")
+	}
+
+	return p.(*Profile), newProfile, nil
+}
+
+// getProfile fetches the profile for the given scoped ID.
+func getProfile(scopedID string) (profile *Profile, err error) {
+	// Get profile from the database.
+	r, err := profileDB.Get(profilesDBPath + scopedID)
+	if err != nil {
+		return nil, err
+	}
+
+	// Parse and prepare the profile, return the result.
+	return prepProfile(r)
+}
+
+// findProfile searches for a profile with the given linked path. If it cannot
+// find one, it will create a new profile for the given linked path.
+func findProfile(linkedPath string) (profile *Profile, new bool, err error) {
+	// Search the database for a matching profile.
+	it, err := profileDB.Query(
+		query.New(makeProfileKey(SourceLocal, "")).Where(
+			query.Where("LinkedPath", query.SameAs, linkedPath),
+		),
+	)
+	if err != nil {
+		return nil, false, err
+	}
+
+	// Only wait for the first result, or until the query ends.
+	r := <-it.Next
+	// Then cancel the query, should it still be running.
+	it.Cancel()
+
+	// Prep and return an existing profile.
+	if r != nil {
+		profile, err = prepProfile(r)
+		return profile, false, err
+	}
+
+	// If there was no profile in the database, create a new one, and return it.
+	profile = New(SourceLocal, "")
+	profile.LinkedPath = linkedPath
+
+	// Check if the profile should be marked as internal.
+	// This is the case whenever the binary resides within the data root dir.
+	if strings.HasPrefix(linkedPath, dataroot.Root().Dir+string(os.PathSeparator)) {
+		profile.Internal = true
+	}
+
+	return profile, true, nil
+}
+
+func prepProfile(r record.Record) (*Profile, error) {
+	// ensure its a profile
+	profile, err := EnsureProfile(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// prepare config
+	err = profile.prepConfig()
+	if err != nil {
+		log.Warningf("profiles: profile %s has (partly) invalid configuration: %s", profile.ID, err)
+	}
+
+	// parse config
+	err = profile.parseConfig()
+	if err != nil {
+		log.Warningf("profiles: profile %s has (partly) invalid configuration: %s", profile.ID, err)
+	}
+
+	// return parsed profile
+	return profile, nil
+}

profile/profile-layered-provider.go

@@ -0,0 +1,50 @@
+package profile
+
+import (
+	"errors"
+	"strings"
+
+	"github.com/safing/portbase/database"
+
+	"github.com/safing/portbase/database/record"
+	"github.com/safing/portbase/runtime"
+)
+
+const (
+	revisionProviderPrefix = "runtime:layeredProfile/"
+)
+
+var (
+	errProfileNotActive = errors.New("profile not active")
+)
+
+func registerRevisionProvider() error {
+	_, err := runtime.DefaultRegistry.Register(
+		revisionProviderPrefix,
+		runtime.SimpleValueGetterFunc(getRevision),
+	)
+	return err
+}
+
+func getRevision(key string) ([]record.Record, error) {
+	key = strings.TrimPrefix(key, revisionProviderPrefix)
+
+	// Get active profile.
+	profile := getActiveProfile(key)
+	if profile == nil {
+		return nil, errProfileNotActive
+	}
+
+	// Get layered profile.
+	layeredProfile := profile.LayeredProfile()
+	if layeredProfile == nil {
+		return nil, database.ErrNotFound
+	}
+
+	// Update profiles if necessary.
+	if layeredProfile.NeedsUpdate() {
+		layeredProfile.Update()
+	}
+
+	return []record.Record{layeredProfile}, nil
+}

profile/profile-layered.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/safing/portbase/database/record"
 	"github.com/safing/portbase/log"
 
 	"github.com/safing/portmaster/status"
@@ -22,10 +23,13 @@ var (
 
 // LayeredProfile combines multiple Profiles.
 type LayeredProfile struct {
-	lock sync.Mutex
+	record.Base
+	sync.RWMutex
 
-	localProfile    *Profile
+	localProfile *Profile
-	layers          []*Profile
+	layers       []*Profile
+
+	LayerIDs        []string
 	RevisionCounter uint64
 
 	validityFlag       *abool.AtomicBool
@@ -34,19 +38,21 @@ type LayeredProfile struct {
 
 	securityLevel *uint32
 
+	// These functions give layered access to configuration options and require
+	// the layered profile to be read locked.
 	DisableAutoPermit   config.BoolOption
 	BlockScopeLocal     config.BoolOption
 	BlockScopeLAN       config.BoolOption
 	BlockScopeInternet  config.BoolOption
 	BlockP2P            config.BoolOption
 	BlockInbound        config.BoolOption
-	EnforceSPN          config.BoolOption
 	RemoveOutOfScopeDNS config.BoolOption
 	RemoveBlockedDNS    config.BoolOption
 	FilterSubDomains    config.BoolOption
 	FilterCNAMEs        config.BoolOption
 	PreventBypassing    config.BoolOption
 	DomainHeuristics    config.BoolOption
+	UseSPN              config.BoolOption
 }
 
 // NewLayeredProfile returns a new layered profile based on the given local profile.
@@ -56,7 +62,8 @@ func NewLayeredProfile(localProfile *Profile) *LayeredProfile {
 	new := &LayeredProfile{
 		localProfile:       localProfile,
 		layers:             make([]*Profile, 0, len(localProfile.LinkedProfiles)+1),
-		revisionCounter:    0,
+		LayerIDs:           make([]string, 0, len(localProfile.LinkedProfiles)+1),
+		RevisionCounter:    0,
 		validityFlag:       abool.NewBool(true),
 		globalValidityFlag: config.NewValidityFlag(),
 		securityLevel:      &securityLevelVal,
@@ -86,10 +93,6 @@ func NewLayeredProfile(localProfile *Profile) *LayeredProfile {
 		CfgOptionBlockInboundKey,
 		cfgOptionBlockInbound,
 	)
-	new.EnforceSPN = new.wrapSecurityLevelOption(
-		CfgOptionEnforceSPNKey,
-		cfgOptionEnforceSPN,
-	)
 	new.RemoveOutOfScopeDNS = new.wrapSecurityLevelOption(
 		CfgOptionRemoveOutOfScopeDNSKey,
 		cfgOptionRemoveOutOfScopeDNS,
@@ -114,18 +117,46 @@ func NewLayeredProfile(localProfile *Profile) *LayeredProfile {
 		CfgOptionDomainHeuristicsKey,
 		cfgOptionDomainHeuristics,
 	)
+	new.UseSPN = new.wrapBoolOption(
+		CfgOptionUseSPNKey,
+		cfgOptionUseSPN,
+	)
 
-	// TODO: load linked profiles.
+	new.LayerIDs = append(new.LayerIDs, localProfile.ScopedID())
-
-	// FUTURE: load forced company profile
 	new.layers = append(new.layers, localProfile)
-	// FUTURE: load company profile
+
-	// FUTURE: load community profile
+	// TODO: Load additional profiles.
 
 	new.updateCaches()
+
+	new.SetKey(revisionProviderPrefix + localProfile.ID)
 	return new
 }
 
+// LockForUsage locks the layered profile, including all layers individually.
+func (lp *LayeredProfile) LockForUsage() {
+	lp.RLock()
+	for _, layer := range lp.layers {
+		layer.RLock()
+	}
+}
+
+// LockForUsage unlocks the layered profile, including all layers individually.
+func (lp *LayeredProfile) UnlockForUsage() {
+	lp.RUnlock()
+	for _, layer := range lp.layers {
+		layer.RUnlock()
+	}
+}
+
+// LocalProfile returns the local profile associated with this layered profile.
+func (lp *LayeredProfile) LocalProfile() *Profile {
+	lp.RLock()
+	defer lp.RUnlock()
+
+	return lp.localProfile
+}
+
 func (lp *LayeredProfile) getValidityFlag() *abool.AtomicBool {
 	lp.validityFlagLock.Lock()
 	defer lp.validityFlagLock.Unlock()
@@ -138,23 +169,56 @@ func (lp *LayeredProfile) RevisionCnt() (revisionCounter uint64) {
 		return 0
 	}
 
-	lp.lock.Lock()
+	lp.RLock()
-	defer lp.lock.Unlock()
+	defer lp.RUnlock()
 
-	return lp.revisionCounter
+	return lp.RevisionCounter
+}
+
+// MarkStillActive marks all the layers as still active.
+func (lp *LayeredProfile) MarkStillActive() {
+	if lp == nil {
+		return
+	}
+
+	lp.RLock()
+	defer lp.RUnlock()
+
+	for _, layer := range lp.layers {
+		layer.MarkStillActive()
+	}
+}
+
+func (lp *LayeredProfile) NeedsUpdate() (outdated bool) {
+	lp.RLock()
+	defer lp.RUnlock()
+
+	// Check global config state.
+	if !lp.globalValidityFlag.IsValid() {
+		return true
+	}
+
+	// Check config in layers.
+	for _, layer := range lp.layers {
+		if layer.outdated.IsSet() {
+			return true
+		}
+	}
+
+	return false
 }
 
 // Update checks for updated profiles and replaces any outdated profiles.
 func (lp *LayeredProfile) Update() (revisionCounter uint64) {
-	lp.lock.Lock()
+	lp.Lock()
-	defer lp.lock.Unlock()
+	defer lp.Unlock()
 
 	var changed bool
 	for i, layer := range lp.layers {
 		if layer.outdated.IsSet() {
 			changed = true
 			// update layer
-			newLayer, err := GetProfile(layer.Source, layer.ID)
+			newLayer, _, err := GetProfile(layer.Source, layer.ID, layer.LinkedPath)
 			if err != nil {
 				log.Errorf("profiles: failed to update profile %s", layer.ScopedID())
 			} else {
@@ -179,10 +243,10 @@ func (lp *LayeredProfile) Update() (revisionCounter uint64) {
 		lp.updateCaches()
 
 		// bump revision counter
-		lp.revisionCounter++
+		lp.RevisionCounter++
 	}
 
-	return lp.revisionCounter
+	return lp.RevisionCounter
 }
 
 func (lp *LayeredProfile) updateCaches() {
@@ -194,8 +258,6 @@ func (lp *LayeredProfile) updateCaches() {
 		}
 	}
 	atomic.StoreUint32(lp.securityLevel, uint32(newLevel))
-
-	// TODO: ignore community profiles
 }
 
 // MarkUsed marks the localProfile as used.
@@ -203,12 +265,12 @@ func (lp *LayeredProfile) MarkUsed() {
 	lp.localProfile.MarkUsed()
 }
 
-// SecurityLevel returns the highest security level of all layered profiles.
+// SecurityLevel returns the highest security level of all layered profiles. This function is atomic and does not require any locking.
 func (lp *LayeredProfile) SecurityLevel() uint8 {
 	return uint8(atomic.LoadUint32(lp.securityLevel))
 }
 
-// DefaultAction returns the active default action ID.
+// DefaultAction returns the active default action ID. This functions requires the layered profile to be read locked.
 func (lp *LayeredProfile) DefaultAction() uint8 {
 	for _, layer := range lp.layers {
 		if layer.defaultAction > 0 {
@@ -221,7 +283,7 @@ func (lp *LayeredProfile) DefaultAction() uint8 {
 	return cfgDefaultAction
 }
 
-// MatchEndpoint checks if the given endpoint matches an entry in any of the profiles.
+// MatchEndpoint checks if the given endpoint matches an entry in any of the profiles. This functions requires the layered profile to be read locked.
 func (lp *LayeredProfile) MatchEndpoint(ctx context.Context, entity *intel.Entity) (endpoints.EPResult, endpoints.Reason) {
 	for _, layer := range lp.layers {
 		if layer.endpoints.IsSet() {
@@ -237,7 +299,7 @@ func (lp *LayeredProfile) MatchEndpoint(ctx context.Context, entity *intel.Entit
 	return cfgEndpoints.Match(ctx, entity)
 }
 
-// MatchServiceEndpoint checks if the given endpoint of an inbound connection matches an entry in any of the profiles.
+// MatchServiceEndpoint checks if the given endpoint of an inbound connection matches an entry in any of the profiles. This functions requires the layered profile to be read locked.
 func (lp *LayeredProfile) MatchServiceEndpoint(ctx context.Context, entity *intel.Entity) (endpoints.EPResult, endpoints.Reason) {
 	entity.EnableReverseResolving()
 
@@ -256,7 +318,7 @@ func (lp *LayeredProfile) MatchServiceEndpoint(ctx context.Context, entity *inte
 }
 
 // MatchFilterLists matches the entity against the set of filter
-// lists.
+// lists. This functions requires the layered profile to be read locked.
 func (lp *LayeredProfile) MatchFilterLists(ctx context.Context, entity *intel.Entity) (endpoints.EPResult, endpoints.Reason) {
 	entity.ResolveSubDomainLists(ctx, lp.FilterSubDomains())
 	entity.EnableCNAMECheck(ctx, lp.FilterCNAMEs())
@@ -287,16 +349,6 @@ func (lp *LayeredProfile) MatchFilterLists(ctx context.Context, entity *intel.En
 	return endpoints.NoMatch, nil
 }
 
-// AddEndpoint adds an endpoint to the local endpoint list, saves the local profile and reloads the configuration.
-func (lp *LayeredProfile) AddEndpoint(newEntry string) {
-	lp.localProfile.AddEndpoint(newEntry)
-}
-
-// AddServiceEndpoint adds a service endpoint to the local endpoint list, saves the local profile and reloads the configuration.
-func (lp *LayeredProfile) AddServiceEndpoint(newEntry string) {
-	lp.localProfile.AddServiceEndpoint(newEntry)
-}
-
 func (lp *LayeredProfile) wrapSecurityLevelOption(configKey string, globalConfig config.IntOption) config.BoolOption {
 	activeAtLevels := lp.wrapIntOption(configKey, globalConfig)
 
@@ -308,6 +360,33 @@ func (lp *LayeredProfile) wrapSecurityLevelOption(configKey string, globalConfig
 	}
 }
 
+func (lp *LayeredProfile) wrapBoolOption(configKey string, globalConfig config.BoolOption) config.BoolOption {
+	valid := no
+	var value bool
+
+	return func() bool {
+		if !valid.IsSet() {
+			valid = lp.getValidityFlag()
+
+			found := false
+		layerLoop:
+			for _, layer := range lp.layers {
+				layerValue, ok := layer.configPerspective.GetAsBool(configKey)
+				if ok {
+					found = true
+					value = layerValue
+					break layerLoop
+				}
+			}
+			if !found {
+				value = globalConfig()
+			}
+		}
+
+		return value
+	}
+}
+
 func (lp *LayeredProfile) wrapIntOption(configKey string, globalConfig config.IntOption) config.IntOption {
 	valid := no
 	var value int64
@@ -335,6 +414,20 @@ func (lp *LayeredProfile) wrapIntOption(configKey string, globalConfig config.In
 	}
 }
 
+// GetProfileSource returns the database key of the first profile in the
+// layers that has the given configuration key set. If it returns an empty
+// string, the global profile can be assumed to have been effective.
+func (lp *LayeredProfile) GetProfileSource(configKey string) string {
+	for _, layer := range lp.layers {
+		if layer.configPerspective.Has(configKey) {
+			return layer.Key()
+		}
+	}
+
+	// Global Profile
+	return ""
+}
+
 /*
 For later:
 

profile/profile.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/tevino/abool"
@@ -53,7 +54,8 @@ const (
 // Profile is used to predefine a security profile for applications.
 type Profile struct { //nolint:maligned // not worth the effort
 	record.Base
-	sync.Mutex
+	sync.RWMutex
+
 	// ID is a unique identifier for the profile.
 	ID string
 	// Source describes the source of the profile.
@@ -73,7 +75,6 @@ type Profile struct { //nolint:maligned // not worth the effort
 	Icon string
 	// IconType describes the type of the Icon property.
 	IconType iconType
-	// References - local profiles only
 	// LinkedPath is a filesystem path to the executable this
 	// profile was created for.
 	LinkedPath string
@@ -99,6 +100,17 @@ type Profile struct { //nolint:maligned // not worth the effort
 	// profile has been created.
 	Created int64
 
+	// Internal is set to true if the profile is attributed to a
+	// Portmaster internal process. Internal is set during profile
+	// creation and may be accessed without lock.
+	Internal bool
+
+	// layeredProfile is a link to the layered profile with this profile as the
+	// main profile.
+	// All processes with the same binary should share the same instance of the
+	// local profile and the associated layered profile.
+	layeredProfile *LayeredProfile
+
 	// Interpreted Data
 	configPerspective *config.Perspective
 	dataParsed        bool
@@ -108,8 +120,9 @@ type Profile struct { //nolint:maligned // not worth the effort
 	filterListIDs     []string
 
 	// Lifecycle Management
-	outdated *abool.AtomicBool
+	usedBy     *LayeredProfile
-	lastUsed time.Time
+	outdated   *abool.AtomicBool
+	lastActive *int64
 
 	internalSave bool
 }
@@ -118,6 +131,7 @@ func (profile *Profile) prepConfig() (err error) {
 	// prepare configuration
 	profile.configPerspective, err = config.NewPerspective(profile.Config)
 	profile.outdated = abool.New()
+	profile.lastActive = new(int64)
 	return
 }
 
@@ -177,16 +191,24 @@ func (profile *Profile) parseConfig() error {
 }
 
 // New returns a new Profile.
-func New() *Profile {
+func New(source profileSource, id string) *Profile {
 	profile := &Profile{
-		ID:           utils.RandomUUID("").String(),
+		ID:           id,
-		Source:       SourceLocal,
+		Source:       source,
 		Created:      time.Now().Unix(),
 		Config:       make(map[string]interface{}),
 		internalSave: true,
 	}
 
-	// create placeholders
+	// Generate random ID if none is given.
+	if id == "" {
+		profile.ID = utils.RandomUUID("").String()
+	}
+
+	// Make key from ID and source.
+	profile.makeKey()
+
+	// Prepare profile to create placeholders.
 	_ = profile.prepConfig()
 	_ = profile.parseConfig()
 
@@ -198,6 +220,11 @@ func (profile *Profile) ScopedID() string {
 	return makeScopedID(profile.Source, profile.ID)
 }
 
+// makeKey derives and sets the record Key from the profile attributes.
+func (profile *Profile) makeKey() {
+	profile.SetKey(makeProfileKey(profile.Source, profile.ID))
+}
+
 // Save saves the profile to the database
 func (profile *Profile) Save() error {
 	if profile.ID == "" {
@@ -207,38 +234,41 @@ func (profile *Profile) Save() error {
 		return fmt.Errorf("profile: profile %s does not specify a source", profile.ID)
 	}
 
-	if !profile.KeyIsSet() {
-		profile.SetKey(makeProfileKey(profile.Source, profile.ID))
-	}
-
 	return profileDB.Put(profile)
 }
 
-// MarkUsed marks the profile as used and saves it when it has changed.
+// MarkStillActive marks the profile as still active.
-func (profile *Profile) MarkUsed() {
+func (profile *Profile) MarkStillActive() {
-	profile.Lock()
+	atomic.StoreInt64(profile.lastActive, time.Now().Unix())
-	// lastUsed
+}
-	profile.lastUsed = time.Now()
+
+// LastActive returns the unix timestamp when the profile was last marked as
+// still active.
+func (profile *Profile) LastActive() int64 {
+	return atomic.LoadInt64(profile.lastActive)
+}
+
+// MarkUsed updates ApproxLastUsed when it's been a while and saves the profile if it was changed.
+func (profile *Profile) MarkUsed() (changed bool) {
+	profile.Lock()
+	defer profile.Unlock()
 
-	// ApproxLastUsed
-	save := false
 	if time.Now().Add(-lastUsedUpdateThreshold).Unix() > profile.ApproxLastUsed {
 		profile.ApproxLastUsed = time.Now().Unix()
-		save = true
+		return true
 	}
-	profile.Unlock()
 
-	if save {
+	return false
-		err := profile.Save()
-		if err != nil {
-			log.Warningf("profiles: failed to save profile %s after marking as used: %s", profile.ScopedID(), err)
-		}
-	}
 }
 
 // String returns a string representation of the Profile.
 func (profile *Profile) String() string {
-	return profile.Name
+	return fmt.Sprintf("<%s %s/%s>", profile.Name, profile.Source, profile.ID)
+}
+
+// IsOutdated returns whether the this instance of the profile is marked as outdated.
+func (profile *Profile) IsOutdated() bool {
+	return profile.outdated.IsSet()
 }
 
 // AddEndpoint adds an endpoint to the endpoint list, saves the profile and reloads the configuration.
@@ -252,82 +282,50 @@ func (profile *Profile) AddServiceEndpoint(newEntry string) {
 }
 
 func (profile *Profile) addEndpointyEntry(cfgKey, newEntry string) {
+	// When finished, save the profile.
+	defer func() {
+		err := profile.Save()
+		if err != nil {
+			log.Warningf("profile: failed to save profile %s after add an endpoint rule: %s", profile.ScopedID(), err)
+		}
+	}()
+
+	// When finished increase the revision counter of the layered profile.
+	defer func() {
+		if profile.layeredProfile != nil {
+			profile.layeredProfile.Lock()
+			defer profile.layeredProfile.Unlock()
+
+			profile.layeredProfile.RevisionCounter++
+		}
+	}()
+
+	// Lock the profile for editing.
 	profile.Lock()
-	// get, update, save endpoints list
+	defer profile.Unlock()
+
+	// Get the endpoint list configuration value and add the new entry.
 	endpointList, ok := profile.configPerspective.GetAsStringArray(cfgKey)
 	if !ok {
 		endpointList = make([]string, 0, 1)
 	}
-	endpointList = append(endpointList, newEntry)
+	endpointList = append([]string{newEntry}, endpointList...)
 	config.PutValueIntoHierarchicalConfig(profile.Config, cfgKey, endpointList)
 
-	profile.Unlock()
+	// Reload the profile manually in order to parse the newly added entry.
-	err := profile.Save()
-	if err != nil {
-		log.Warningf("profile: failed to save profile after adding endpoint: %s", err)
-	}
-
-	// reload manually
-	profile.Lock()
 	profile.dataParsed = false
-	err = profile.parseConfig()
+	err := profile.parseConfig()
 	if err != nil {
-		log.Warningf("profile: failed to parse profile config after adding endpoint: %s", err)
+		log.Warningf("profile: failed to parse %s config after adding endpoint: %s", profile, err)
 	}
-	profile.Unlock()
 }
 
-// GetProfile loads a profile from the database.
+// LayeredProfile returns the layered profile associated with this profile.
-func GetProfile(source profileSource, id string) (*Profile, error) {
+func (profile *Profile) LayeredProfile() *LayeredProfile {
-	return GetProfileByScopedID(makeScopedID(source, id))
-}
-
-// GetProfileByScopedID loads a profile from the database using a scoped ID like "local/id" or "community/id".
-func GetProfileByScopedID(scopedID string) (*Profile, error) {
-	// check cache
-	profile := getActiveProfile(scopedID)
-	if profile != nil {
-		profile.MarkUsed()
-		return profile, nil
-	}
-
-	// get from database
-	r, err := profileDB.Get(profilesDBPath + scopedID)
-	if err != nil {
-		return nil, err
-	}
-
-	// convert
-	profile, err = EnsureProfile(r)
-	if err != nil {
-		return nil, err
-	}
-
-	// lock for prepping
 	profile.Lock()
+	defer profile.Unlock()
 
-	// prepare config
+	return profile.layeredProfile
-	err = profile.prepConfig()
-	if err != nil {
-		log.Warningf("profiles: profile %s has (partly) invalid configuration: %s", profile.ID, err)
-	}
-
-	// parse config
-	err = profile.parseConfig()
-	if err != nil {
-		log.Warningf("profiles: profile %s has (partly) invalid configuration: %s", profile.ID, err)
-	}
-
-	// mark as internal
-	profile.internalSave = true
-
-	profile.Unlock()
-
-	// mark active
-	profile.MarkUsed()
-	markProfileActive(profile)
-
-	return profile, nil
 }
 
 // EnsureProfile ensures that the given record is a *Profile, and returns it.

profile/special.go

@@ -1,56 +0,0 @@
-package profile
-
-import (
-	"github.com/safing/portbase/log"
-)
-
-const (
-	unidentifiedProfileID = "_unidentified"
-	systemProfileID       = "_system"
-)
-
-// GetUnidentifiedProfile returns the special profile assigned to unidentified processes.
-func GetUnidentifiedProfile() *Profile {
-	// get profile
-	profile, err := GetProfile(SourceLocal, unidentifiedProfileID)
-	if err == nil {
-		return profile
-	}
-
-	// create if not available (or error)
-	profile = New()
-	profile.Name = "Unidentified Processes"
-	profile.Source = SourceLocal
-	profile.ID = unidentifiedProfileID
-
-	// save to db
-	err = profile.Save()
-	if err != nil {
-		log.Warningf("profiles: failed to save %s: %s", profile.ScopedID(), err)
-	}
-
-	return profile
-}
-
-// GetSystemProfile returns the special profile used for the Kernel.
-func GetSystemProfile() *Profile {
-	// get profile
-	profile, err := GetProfile(SourceLocal, systemProfileID)
-	if err == nil {
-		return profile
-	}
-
-	// create if not available (or error)
-	profile = New()
-	profile.Name = "Operating System"
-	profile.Source = SourceLocal
-	profile.ID = systemProfileID
-
-	// save to db
-	err = profile.Save()
-	if err != nil {
-		log.Warningf("profiles: failed to save %s: %s", profile.ScopedID(), err)
-	}
-
-	return profile
-}