Add scoping to IPInfo

firewall/dns.go

@@ -205,10 +205,21 @@ func mayBlockCNAMEs(conn *network.Connection) bool {
 	return false
 }
 
+// updateIPsAndCNAMEs saves all the IP->Name mappings to the cache database and
+// updates the CNAMEs in the Connection's Entity.
 func updateIPsAndCNAMEs(q *resolver.Query, rrCache *resolver.RRCache, conn *network.Connection) {
-	// save IP addresses to IPInfo
+	// FIXME: ignore localhost
+
+	// Get IPInfo scope.
+	var scope string
+	proc := conn.Process()
+	if proc != nil {
+		scope = proc.LocalProfileKey
+	}
+
+	// Collect IPs and CNAMEs.
 	cnames := make(map[string]string)
-	ips := make(map[string]struct{})
+	ips := make([]net.IP, 0, len(rrCache.Answer))
 
 	for _, rr := range append(rrCache.Answer, rrCache.Extra...) {
 		switch v := rr.(type) {
@@ -216,19 +227,27 @@ func updateIPsAndCNAMEs(q *resolver.Query, rrCache *resolver.RRCache, conn *netw
 			cnames[v.Hdr.Name] = v.Target
 
 		case *dns.A:
-			ips[v.A.String()] = struct{}{}
+			ips = append(ips, v.A)
 
 		case *dns.AAAA:
-			ips[v.AAAA.String()] = struct{}{}
+			ips = append(ips, v.AAAA)
 		}
 	}
 
-	for ip := range ips {
+	// Package IPs and CNAMEs into IPInfo structs.
-		record := resolver.ResolvedDomain{
+	for _, ip := range ips {
-			Domain: q.FQDN,
+		// Never save domain attributions for localhost IPs.
+		if netutils.ClassifyIP(ip) == netutils.HostLocal {
+			continue
 		}
 
-		// resolve all CNAMEs in the correct order.
+		// Create new record for this IP.
+		record := resolver.ResolvedDomain{
+			Domain:  q.FQDN,
+			Expires: rrCache.TTL,
+		}
+
+		// Resolve all CNAMEs in the correct order and add the to the record.
 		var domain = q.FQDN
 		for {
 			nextDomain, isCNAME := cnames[domain]
@@ -240,31 +259,30 @@ func updateIPsAndCNAMEs(q *resolver.Query, rrCache *resolver.RRCache, conn *netw
 			domain = nextDomain
 		}
 
-		// update the entity to include the cnames
+		// Update the entity to include the CNAMEs of the query response.
 		conn.Entity.CNAME = record.CNAMEs
 
-		// get the existing IP info or create a new  one
+		// Check if there is an existing record for this DNS response.
-		var save bool
+		// Else create a new one.
-		info, err := resolver.GetIPInfo(ip)
+		ipString := ip.String()
+		info, err := resolver.GetIPInfo(scope, ipString)
 		if err != nil {
 			if err != database.ErrNotFound {
 				log.Errorf("nameserver: failed to search for IP info record: %s", err)
 			}
 
 			info = &resolver.IPInfo{
-				IP: ip,
+				IP:    ipString,
+				Scope: scope,
 			}
-			save = true
 		}
 
-		// and the new resolved domain record and save
+		// Add the new record to the resolved domains for this IP and scope.
-		if new := info.AddDomain(record); new {
+		info.AddDomain(record)
-			save = true
+
-		}
+		// Save if the record is new or has been updated.
-		if save {
+		if err := info.Save(); err != nil {
-			if err := info.Save(); err != nil {
+			log.Errorf("nameserver: failed to save IP info record: %s", err)
-				log.Errorf("nameserver: failed to save IP info record: %s", err)
-			}
 		}
 	}
 }

network/connection.go

@@ -139,9 +139,9 @@ func NewConnectionFromFirstPacket(pkt packet.Packet) *Connection {
 		}
 
 		// check if we can find a domain for that IP
-		ipinfo, err := resolver.GetIPInfo(pkt.Info().Dst.String())
+		ipinfo, err := resolver.GetIPInfo(proc.LocalProfileKey, pkt.Info().Dst.String())
 		if err == nil {
-			lastResolvedDomain := ipinfo.ResolvedDomains.MostRecentDomain()
+			lastResolvedDomain := ipinfo.MostRecentDomain()
 			if lastResolvedDomain != nil {
 				scope = lastResolvedDomain.Domain
 				entity.Domain = lastResolvedDomain.Domain

resolver/ipinfo.go

@@ -7,12 +7,25 @@ import (
 
 	"github.com/safing/portbase/database"
 	"github.com/safing/portbase/database/record"
-	"github.com/safing/portbase/utils"
+)
+
+const (
+	IPInfoScopeGlobal = "global"
 )
 
 var (
 	ipInfoDatabase = database.NewInterface(&database.Options{
-		AlwaysSetRelativateExpiry: 86400, // 24 hours
+		Local:    true,
+		Internal: true,
+
+		// Cache entries because new/updated entries will often be queries soon
+		// after inserted.
+		CacheSize: 256,
+
+		// We only use the cache database here, so we can delay and batch all our
+		// writes. Also, no one else accesses these records, so we are fine using
+		// this.
+		DelayCachedWrites: "cache",
 	})
 )
 
@@ -25,6 +38,11 @@ type ResolvedDomain struct {
 	// CNAMEs is a list of CNAMEs that have been resolved for
 	// Domain.
 	CNAMEs []string
+
+	// Expires holds the timestamp when this entry expires.
+	// This does not mean that the entry may not be used anymore afterwards,
+	// but that this is used to calcuate the TTL of the database record.
+	Expires int64
 }
 
 // String returns a string representation of ResolvedDomain including
@@ -54,29 +72,17 @@ func (rds ResolvedDomains) String() string {
 	return strings.Join(domains, " or ")
 }
 
-// MostRecentDomain returns the most recent domain.
-func (rds ResolvedDomains) MostRecentDomain() *ResolvedDomain {
-	if len(rds) == 0 {
-		return nil
-	}
-	// TODO(ppacher): we could also do that by using ResolvedAt()
-	mostRecent := rds[len(rds)-1]
-	return &mostRecent
-}
-
 // IPInfo represents various information about an IP.
 type IPInfo struct {
 	record.Base
 	sync.Mutex
 
-	// IP holds the acutal IP address.
+	// IP holds the actual IP address.
 	IP string
 
-	// Domains holds a list of domains that have been
+	// Scope holds a scope for this IPInfo.
-	// resolved to IP. This field is deprecated and should
+	// Usually this would be the Profile ID of the associated process.
-	// be removed.
+	Scope string
-	// DEPRECATED: remove with alpha.
-	Domains []string `json:"Domains,omitempty"`
 
 	// ResolvedDomain is a slice of domains that
 	// have been requested by various applications
@@ -84,35 +90,43 @@ type IPInfo struct {
 	ResolvedDomains ResolvedDomains
 }
 
-// AddDomain adds a new resolved domain to ipi.
+// AddDomain adds a new resolved domain to IPInfo.
-func (ipi *IPInfo) AddDomain(resolved ResolvedDomain) bool {
+func (info *IPInfo) AddDomain(resolved ResolvedDomain) {
-	for idx, d := range ipi.ResolvedDomains {
+	info.Lock()
-		if d.Domain == resolved.Domain {
+	defer info.Unlock()
-			if utils.StringSliceEqual(d.CNAMEs, resolved.CNAMEs) {
-				return false
-			}
 
-			// we have a different CNAME chain now, remove the previous
+	// Delete old for the same domain.
-			// entry and add it at the end.
+	for idx, d := range info.ResolvedDomains {
-			ipi.ResolvedDomains = append(ipi.ResolvedDomains[:idx], ipi.ResolvedDomains[idx+1:]...)
+		if d.Domain == resolved.Domain {
-			ipi.ResolvedDomains = append(ipi.ResolvedDomains, resolved)
+			info.ResolvedDomains = append(info.ResolvedDomains[:idx], info.ResolvedDomains[idx+1:]...)
-			return true
+			break
 		}
 	}
 
-	ipi.ResolvedDomains = append(ipi.ResolvedDomains, resolved)
+	// Add new entry to the end.
-	return true
+	info.ResolvedDomains = append(info.ResolvedDomains, resolved)
 }
 
-func makeIPInfoKey(ip string) string {
+// MostRecentDomain returns the most recent domain.
-	return fmt.Sprintf("cache:intel/ipInfo/%s", ip)
+func (info *IPInfo) MostRecentDomain() *ResolvedDomain {
+	info.Lock()
+	defer info.Unlock()
+
+	if len(info.ResolvedDomains) == 0 {
+		return nil
+	}
+
+	mostRecent := info.ResolvedDomains[len(info.ResolvedDomains)-1]
+	return &mostRecent
+}
+
+func makeIPInfoKey(scope, ip string) string {
+	return fmt.Sprintf("cache:intel/ipInfo/%s/%s", scope, ip)
 }
 
 // GetIPInfo gets an IPInfo record from the database.
-func GetIPInfo(ip string) (*IPInfo, error) {
+func GetIPInfo(scope, ip string) (*IPInfo, error) {
-	key := makeIPInfoKey(ip)
+	r, err := ipInfoDatabase.Get(makeIPInfoKey(scope, ip))
-
-	r, err := ipInfoDatabase.Get(key)
 	if err != nil {
 		return nil, err
 	}
@@ -126,18 +140,6 @@ func GetIPInfo(ip string) (*IPInfo, error) {
 			return nil, err
 		}
 
-		// Legacy support,
-		// DEPRECATED: remove with alpha
-		if len(new.Domains) > 0 && len(new.ResolvedDomains) == 0 {
-			for _, d := range new.Domains {
-				new.ResolvedDomains = append(new.ResolvedDomains, ResolvedDomain{
-					Domain: d,
-					// rest is empty...
-				})
-			}
-			new.Domains = nil // clean up so we remove it from the database
-		}
-
 		return new, nil
 	}
 
@@ -150,27 +152,38 @@ func GetIPInfo(ip string) (*IPInfo, error) {
 }
 
 // Save saves the IPInfo record to the database.
-func (ipi *IPInfo) Save() error {
+func (info *IPInfo) Save() error {
-	ipi.Lock()
+	info.Lock()
-	if !ipi.KeyIsSet() {
-		ipi.SetKey(makeIPInfoKey(ipi.IP))
-	}
-	ipi.Unlock()
 
-	// Legacy support
+	// Set database key if not yet set already.
-	// Ensure we don't write new Domain fields into the
+	if !info.KeyIsSet() {
-	// database.
+		// Default to global scope if scope is unset.
-	// DEPRECATED: remove with alpha
+		if info.Scope == "" {
-	if len(ipi.Domains) > 0 {
+			info.Scope = IPInfoScopeGlobal
-		ipi.Domains = nil
+		}
+		info.SetKey(makeIPInfoKey(info.Scope, info.IP))
 	}
 
-	return ipInfoDatabase.Put(ipi)
+	// Calculate and set cache expiry.
+	var expires int64 = 86400 // Minimum TTL of one day.
+	for _, rd := range info.ResolvedDomains {
+		if rd.Expires > expires {
+			expires = rd.Expires
+		}
+	}
+	info.UpdateMeta()
+	expires += 3600 // Add one hour to expiry as a buffer.
+	info.Meta().SetAbsoluteExpiry(expires)
+
+	info.Unlock()
+
+	return ipInfoDatabase.Put(info)
 }
 
 // FmtDomains returns a string consisting of the domains that have seen to use this IP, joined by " or "
-func (ipi *IPInfo) String() string {
+func (info *IPInfo) String() string {
-	ipi.Lock()
+	info.Lock()
-	defer ipi.Unlock()
+	defer info.Unlock()
-	return fmt.Sprintf("<IPInfo[%s] %s: %s", ipi.Key(), ipi.IP, ipi.ResolvedDomains.String())
+
+	return fmt.Sprintf("<IPInfo[%s] %s: %s>", info.Key(), info.IP, info.ResolvedDomains.String())
 }

resolver/ipinfo_test.go

@@ -15,7 +15,7 @@ func TestIPInfo(t *testing.T) {
 		CNAMEs: []string{"example.com"},
 	}
 
-	ipi := &IPInfo{
+	info := &IPInfo{
 		IP: "1.2.3.4",
 		ResolvedDomains: ResolvedDomains{
 			example,
@@ -27,22 +27,18 @@ func TestIPInfo(t *testing.T) {
 		Domain: "sub2.example.com",
 		CNAMEs: []string{"sub1.example.com", "example.com"},
 	}
-	added := ipi.AddDomain(sub2Example)
+	info.AddDomain(sub2Example)
-
+	assert.Equal(t, ResolvedDomains{example, subExample, sub2Example}, info.ResolvedDomains)
-	assert.True(t, added)
-	assert.Equal(t, ResolvedDomains{example, subExample, sub2Example}, ipi.ResolvedDomains)
 
 	// try again, should do nothing now
-	added = ipi.AddDomain(sub2Example)
+	info.AddDomain(sub2Example)
-	assert.False(t, added)
+	assert.Equal(t, ResolvedDomains{example, subExample, sub2Example}, info.ResolvedDomains)
-	assert.Equal(t, ResolvedDomains{example, subExample, sub2Example}, ipi.ResolvedDomains)
 
 	subOverWrite := ResolvedDomain{
 		Domain: "sub1.example.com",
 		CNAMEs: []string{}, // now without CNAMEs
 	}
 
-	added = ipi.AddDomain(subOverWrite)
+	info.AddDomain(subOverWrite)
-	assert.True(t, added)
+	assert.Equal(t, ResolvedDomains{example, sub2Example, subOverWrite}, info.ResolvedDomains)
-	assert.Equal(t, ResolvedDomains{example, sub2Example, subOverWrite}, ipi.ResolvedDomains)
 }

resolver/main.go

@@ -93,6 +93,9 @@ func start() error {
 		listenToMDNS,
 	)
 
+	module.StartServiceWorker("name record delayed cache writer", 0, recordDatabase.DelayedCacheWriter)
+	module.StartServiceWorker("ip info delayed cache writer", 0, ipInfoDatabase.DelayedCacheWriter)
+
 	return nil
 }
 

resolver/namerecord.go

@@ -12,10 +12,24 @@ import (
 	"github.com/safing/portbase/log"
 )
 
+const (
+	// databaseOvertime defines how much longer than the TTL name records are
+	// cached in the database.
+	databaseOvertime = 86400 * 14 // two weeks
+)
+
 var (
 	recordDatabase = database.NewInterface(&database.Options{
-		AlwaysSetRelativateExpiry: 2592000, // 30 days
+		Local:    true,
-		CacheSize:                 256,
+		Internal: true,
+
+		// Cache entries because application often resolve domains multiple times.
+		CacheSize: 256,
+
+		// We only use the cache database here, so we can delay and batch all our
+		// writes. Also, no one else accesses these records, so we are fine using
+		// this.
+		DelayCachedWrites: "cache",
 	})
 
 	nameRecordsKeyPrefix = "cache:intel/nameRecord/"
@@ -32,7 +46,8 @@ type NameRecord struct {
 	Answer   []string
 	Ns       []string
 	Extra    []string
-	TTL      int64
+	// TODO: Name change in progress. Rename "TTL" field to "Expires" in Q1 2021.
+	TTL int64 `json:"Expires"`
 
 	Server      string
 	ServerScope int8
@@ -84,6 +99,9 @@ func (rec *NameRecord) Save() error {
 	}
 
 	rec.SetKey(makeNameRecordKey(rec.Domain, rec.Question))
+	rec.UpdateMeta()
+	rec.Meta().SetAbsoluteExpiry(rec.TTL + databaseOvertime)
+
 	return recordDatabase.PutNew(rec)
 }
 

resolver/rrcache.go

@@ -28,7 +28,8 @@ type RRCache struct {
 	Answer []dns.RR
 	Ns     []dns.RR
 	Extra  []dns.RR
-	TTL    int64
+	// TODO: Name change in progress. Rename "TTL" field to "Expires" in Q1 2021.
+	TTL int64 `json:"Expires"`
 
 	// Source Information
 	Server      string