Merge pull request #124 from safing/fix/lms-scoring

Move LMS scoring from nameserver to firewall

README.md

@@ -35,8 +35,8 @@ Think of a pi-hole for your computer. Or an ad-blocker that blocks ads on your w
 **Features/Settings:**
 
 - Select and activate block-lists
-- Manually black/whitelist domains
-  - You can whitelist domains in case something breaks
+- Manually block/allow domains
+  - You can allow domains in case something breaks
 - CNAME Blocking (block these new nasty "unblockable" ads/trackers)
 - Block all subdomains of a domain in the block-lists
 

detection/dga/lms.go

@@ -4,16 +4,13 @@ import (
 	"strings"
 )
 
-// LmsScoreOfDomain calculates the mean longest meaningful substring of a domain. It follows some special rules to increase accuracy. It returns a value between 0 and 100, representing the length-based percentage of the meaningful substring.
+// LmsScoreOfDomain calculates the mean longest meaningful substring of a domain.
+// It follows some special rules to increase accuracy. It returns a value between
+// 0 and 100, representing the length-based percentage of the meaningful substring.
 func LmsScoreOfDomain(domain string) float64 {
 	var totalScore float64
 	domain = strings.ToLower(domain)
 	subjects := strings.Split(domain, ".")
-	// ignore the last two parts
-	if len(subjects) <= 3 {
-		return 100
-	}
-	subjects = subjects[:len(subjects)-3]
 	var totalLength int
 	for _, subject := range subjects {
 		totalLength += len(subject)
@@ -27,7 +24,9 @@ func LmsScoreOfDomain(domain string) float64 {
 	return totalScore
 }
 
-// LmsScore calculates the longest meaningful substring of a domain. It returns a value between 0 and 100, representing the length-based percentage of the meaningful substring.
+// LmsScore calculates the longest meaningful substring of a domain. It returns a
+// value between 0 and 100, representing the length-based percentage of the
+// meaningful substring.
 func LmsScore(subject string) float64 {
 	lmsStart := -1
 	lmsStop := -1

detection/dga/lms_test.go

@@ -5,8 +5,8 @@ import "testing"
 func TestLmsScoreOfDomain(t *testing.T) {
 	testDomain(t, "g.symcd.com.", 100, 100)
 	testDomain(t, "www.google.com.", 100, 100)
-	testDomain(t, "55ttt5.12abc3.test.com.", 50, 50)
-	testDomain(t, "mbtq6opnuodp34gcrma65fxacgxv5ukr7lq6xuhr4mhoibe7.yvqptrozfbnqyemchpovw3q5xwjibuxfsgb72mix3znhpfhc.i2n7jh2gadqaadck3zs3vg3hbv5pkmwzeay4gc75etyettbb.isi5mhmowtfriu33uxzmgvjur5g2p3tloynwohfrggee6fkn.meop7kqyd5gwxxa3.er.spotify.com.", 0, 30)
+	testDomain(t, "55ttt5.12abc3.test.com.", 68, 69)
+	testDomain(t, "mbtq6opnuodp34gcrma65fxacgxv5ukr7lq6xuhr4mhoibe7.yvqptrozfbnqyemchpovw3q5xwjibuxfsgb72mix3znhpfhc.i2n7jh2gadqaadck3zs3vg3hbv5pkmwzeay4gc75etyettbb.isi5mhmowtfriu33uxzmgvjur5g2p3tloynwohfrggee6fkn.meop7kqyd5gwxxa3.er.spotify.com.", 0, 31)
 }
 
 func testDomain(t *testing.T, domain string, min, max float64) {

firewall/master.go

@@ -7,7 +7,9 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/safing/portmaster/detection/dga"
 	"github.com/safing/portmaster/netenv"
+	"golang.org/x/net/publicsuffix"
 
 	"github.com/safing/portbase/log"
 	"github.com/safing/portmaster/network"
@@ -58,6 +60,7 @@ func DecideOnConnection(ctx context.Context, conn *network.Connection, pkt packe
 		checkBypassPrevention,
 		checkFilterLists,
 		checkInbound,
+		checkDomainHeuristics,
 		checkDefaultPermit,
 		checkAutoPermitRelated,
 		checkDefaultAction,
@@ -70,7 +73,7 @@ func DecideOnConnection(ctx context.Context, conn *network.Connection, pkt packe
 	}
 
 	// DefaultAction == DefaultActionBlock
-	conn.Deny("endpoint is not whitelisted (default=block)")
+	conn.Deny("endpoint is not allowed (default=block)")
 }
 
 // checkPortmasterConnection allows all connection that originate from
@@ -281,10 +284,70 @@ func checkFilterLists(ctx context.Context, conn *network.Connection, pkt packet.
 	return false
 }
 
+func checkDomainHeuristics(ctx context.Context, conn *network.Connection, _ packet.Packet) bool {
+	p := conn.Process().Profile()
+
+	if !p.DomainHeuristics() {
+		return false
+	}
+
+	if conn.Entity.Domain == "" {
+		return false
+	}
+
+	trimmedDomain := strings.TrimRight(conn.Entity.Domain, ".")
+	etld1, err := publicsuffix.EffectiveTLDPlusOne(trimmedDomain)
+	if err != nil {
+		// we don't apply any checks here and let the request through
+		// because a malformed domain-name will likely be dropped by
+		// checks better suited for that.
+		log.Tracer(ctx).Warningf("nameserver: failed to get eTLD+1: %s", err)
+		return false
+	}
+
+	domainToCheck := strings.Split(etld1, ".")[0]
+	score := dga.LmsScore(domainToCheck)
+	if score < 5 {
+		log.Tracer(ctx).Warningf(
+			"nameserver: possible data tunnel by %s in eTLD+1 %s: %s has an lms score of %.2f, returning nxdomain",
+			conn.Process(),
+			etld1,
+			domainToCheck,
+			score,
+		)
+		conn.Block("possible DGA domain commonly used by malware")
+		return true
+	}
+	log.Tracer(ctx).Infof("LMS score of eTLD+1 %s is %.2f", etld1, score)
+
+	// 100 is a somewhat arbitrary threshold to ensure we don't mess
+	// around with CDN domain names to early. They use short second-level
+	// domains that would trigger LMS checks but are to small to actually
+	// exfiltrate data.
+	if len(conn.Entity.Domain) > len(etld1)+100 {
+		domainToCheck = trimmedDomain[0:len(etld1)]
+		score := dga.LmsScoreOfDomain(domainToCheck)
+		if score < 10 {
+			log.Tracer(ctx).Warningf(
+				"nameserver: possible data tunnel by %s in subdomain %s: %s has an lms score of %.2f, returning nxdomain",
+				conn.Process(),
+				conn.Entity.Domain,
+				domainToCheck,
+				score,
+			)
+			conn.Block("possible data tunnel for covert communication and protection bypassing")
+			return true
+		}
+		log.Tracer(ctx).Infof("LMS score of entire domain is %.2f", score)
+	}
+
+	return false
+}
+
 func checkInbound(_ context.Context, conn *network.Connection, _ packet.Packet) bool {
 	// implicit default=block for inbound
 	if conn.Inbound {
-		conn.Drop("endpoint is not whitelisted (incoming is always default=block)")
+		conn.Drop("endpoint is not allowed (incoming is always default=block)")
 		return true
 	}
 	return false
@@ -294,7 +357,7 @@ func checkDefaultPermit(_ context.Context, conn *network.Connection, _ packet.Pa
 	// check default action
 	p := conn.Process().Profile()
 	if p.DefaultAction() == profile.DefaultActionPermit {
-		conn.Accept("endpoint is not blacklisted (default=permit)")
+		conn.Accept("endpoint is not blocked (default=permit)")
 		return true
 	}
 	return false

intel/filterlists/lookup.go

@@ -47,14 +47,14 @@ func lookupBlockLists(entity, value string) ([]string, error) {
 }
 
 // LookupCountry returns a list of sources that mark the country
-// as blacklisted. If country is not stored in the cache database
+// as blocked. If country is not stored in the cache database
 // a nil slice is returned.
 func LookupCountry(country string) ([]string, error) {
 	return lookupBlockLists("country", country)
 }
 
 // LookupDomain returns a list of sources that mark the domain
-// as blacklisted. If domain is not stored in the cache database
+// as blocked. If domain is not stored in the cache database
 // a nil slice is returned.
 func LookupDomain(domain string) ([]string, error) {
 	// make sure we only fully qualified domains
@@ -67,13 +67,13 @@ func LookupDomain(domain string) ([]string, error) {
 }
 
 // LookupASNString returns a list of sources that mark the ASN
-// as blacklisted. If ASN is not stored in the cache database
+// as blocked. If ASN is not stored in the cache database
 // a nil slice is returned.
 func LookupASNString(asn string) ([]string, error) {
 	return lookupBlockLists("asn", asn)
 }
 
-// LookupIP returns a list of blacklist sources that contain
+// LookupIP returns a list of block sources that contain
 // a reference to ip. LookupIP automatically checks the IPv4 or
 // IPv6 lists respectively.
 func LookupIP(ip net.IP) ([]string, error) {
@@ -95,7 +95,7 @@ func LookupIPString(ipStr string) ([]string, error) {
 	return LookupIP(ip)
 }
 
-// LookupIPv4String returns a list of blacklist sources that
+// LookupIPv4String returns a list of block sources that
 // contain a reference to ip. If the IP is not stored in the
 // cache database a nil slice is returned.
 func LookupIPv4String(ipv4 string) ([]string, error) {
@@ -113,7 +113,7 @@ func LookupIPv4(ipv4 net.IP) ([]string, error) {
 	return LookupIPv4String(ip.String())
 }
 
-// LookupIPv6String returns a list of blacklist sources that
+// LookupIPv6String returns a list of block sources that
 // contain a reference to ip. If the IP is not stored in the
 // cache database a nil slice is returned.
 func LookupIPv6String(ipv6 string) ([]string, error) {

nameserver/nameserver.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/modules"
-	"github.com/safing/portmaster/detection/dga"
 	"github.com/safing/portmaster/firewall"
 	"github.com/safing/portmaster/nameserver/nsutil"
 	"github.com/safing/portmaster/netenv"
@@ -211,17 +210,6 @@ func handleRequest(ctx context.Context, w dns.ResponseWriter, query *dns.Msg) er
 	// save security level to query
 	q.SecurityLevel = conn.Process().Profile().SecurityLevel()
 
-	// check for possible DNS tunneling / data transmission
-	// TODO: improve this
-	lms := dga.LmsScoreOfDomain(q.FQDN)
-	// log.Tracef("nameserver: domain %s has lms score of %f", fqdn, lms)
-	if lms < 10 {
-		tracer.Warningf("nameserver: possible data tunnel by %s: %s has lms score of %f, returning nxdomain", conn.Process(), q.FQDN, lms)
-		conn.Block("Possible data tunnel")
-		sendResponse(w, query, conn.Verdict, conn.Reason, conn.ReasonContext)
-		return nil
-	}
-
 	// check profile before we even get intel and rr
 	firewall.DecideOnConnection(ctx, conn, nil)
 

network/connection.go

@@ -87,10 +87,10 @@ func NewConnectionFromDNSRequest(ctx context.Context, fqdn string, cnames []stri
 	timestamp := time.Now().Unix()
 	dnsConn := &Connection{
 		Scope: fqdn,
-		Entity: (&intel.Entity{
+		Entity: &intel.Entity{
 			Domain: fqdn,
 			CNAME:  cnames,
-		}),
+		},
 		process: proc,
 		Started: timestamp,
 		Ended:   timestamp,
@@ -123,20 +123,20 @@ func NewConnectionFromFirstPacket(pkt packet.Packet) *Connection {
 		default: // netutils.Invalid
 			scope = IncomingInvalid
 		}
-		entity = (&intel.Entity{
+		entity = &intel.Entity{
 			IP:       pkt.Info().Src,
 			Protocol: uint8(pkt.Info().Protocol),
 			Port:     pkt.Info().SrcPort,
-		})
+		}
 
 	} else {
 
 		// outbound connection
-		entity = (&intel.Entity{
+		entity = &intel.Entity{
 			IP:       pkt.Info().Dst,
 			Protocol: uint8(pkt.Info().Protocol),
 			Port:     pkt.Info().DstPort,
-		})
+		}
 
 		// check if we can find a domain for that IP
 		ipinfo, err := resolver.GetIPInfo(pkt.Info().Dst.String())

network/dns.go

@@ -23,15 +23,22 @@ var (
 	unidentifiedProcessScopePrefix = strconv.Itoa(process.UnidentifiedProcessID) + "/"
 )
 
+func getDNSRequestCacheKey(pid int, fqdn string) string {
+	return strconv.Itoa(pid) + "/" + fqdn
+}
+
 func removeOpenDNSRequest(pid int, fqdn string) {
 	openDNSRequestsLock.Lock()
 	defer openDNSRequestsLock.Unlock()
 
-	key := strconv.Itoa(pid) + "/" + fqdn
+	key := getDNSRequestCacheKey(pid, fqdn)
 	_, ok := openDNSRequests[key]
 	if ok {
 		delete(openDNSRequests, key)
-	} else if pid != process.UnidentifiedProcessID {
+		return
+	}
+
+	if pid != process.UnidentifiedProcessID {
 		// check if there is an open dns request from an unidentified process
 		delete(openDNSRequests, unidentifiedProcessScopePrefix+fqdn)
 	}
@@ -42,26 +49,24 @@ func SaveOpenDNSRequest(conn *Connection) {
 	openDNSRequestsLock.Lock()
 	defer openDNSRequestsLock.Unlock()
 
-	key := strconv.Itoa(conn.process.Pid) + "/" + conn.Scope
-
-	existingConn, ok := openDNSRequests[key]
-	if ok {
+	key := getDNSRequestCacheKey(conn.process.Pid, conn.Scope)
+	if existingConn, ok := openDNSRequests[key]; ok {
 		existingConn.Lock()
 		defer existingConn.Unlock()
-
 		existingConn.Ended = conn.Started
-	} else {
-		openDNSRequests[key] = conn
+		return
 	}
+
+	openDNSRequests[key] = conn
 }
 
 func openDNSRequestWriter(ctx context.Context) error {
 	ticker := time.NewTicker(writeOpenDNSRequestsTickDuration)
+	defer ticker.Stop()
 
 	for {
 		select {
 		case <-ctx.Done():
-			ticker.Stop()
 			return nil
 		case <-ticker.C:
 			writeOpenDNSRequestsToDB()

profile/config.go

@@ -80,14 +80,18 @@ var (
 	cfgOptionRemoveBlockedDNS      config.IntOption // security level option
 	cfgOptionRemoveBlockedDNSOrder = 113
 
+	CfgOptionDomainHeuristicsKey   = "filter/domainHeuristics"
+	cfgOptionDomainHeuristics      config.IntOption // security level option
+	cfgOptionDomainHeuristicsOrder = 114
+
 	// Permanent Verdicts Order = 128
 )
 
 func registerConfiguration() error {
 	// Default Filter Action
-	// permit - blacklist mode: everything is permitted unless blocked
+	// permit - blocklist mode: everything is permitted unless blocked
 	// ask - ask mode: if not verdict is found, the user is consulted
-	// block - whitelist mode: everything is blocked unless permitted
+	// block - allowlist mode: everything is blocked unless permitted
 	err := config.Register(&config.Option{
 		Name:            "Default Filter Action",
 		Key:             CfgOptionDefaultActionKey,
@@ -378,6 +382,24 @@ Examples:
 	cfgOptionRemoveBlockedDNS = config.Concurrent.GetAsInt(CfgOptionRemoveBlockedDNSKey, int64(status.SecurityLevelsAll))
 	cfgIntOptions[CfgOptionRemoveBlockedDNSKey] = cfgOptionRemoveBlockedDNS
 
+	// Domain heuristics
+	err = config.Register(&config.Option{
+		Name:            "Enable Domain Heuristics",
+		Key:             CfgOptionDomainHeuristicsKey,
+		Description:     "Domain Heuristics checks for suspicious looking domain names and blocks them. Ths option currently targets domains generated by malware and DNS data tunnels.",
+		Order:           cfgOptionDomainHeuristicsOrder,
+		OptType:         config.OptTypeInt,
+		ExpertiseLevel:  config.ExpertiseLevelExpert,
+		ExternalOptType: "security level",
+		DefaultValue:    status.SecurityLevelsAll,
+		ValidationRegex: "^(0|4|6|7)$",
+	})
+	if err != nil {
+		return err
+	}
+	cfgOptionDomainHeuristics = config.Concurrent.GetAsInt(CfgOptionDomainHeuristicsKey, int64(status.SecurityLevelsAll))
+
+	// Bypass prevention
 	err = config.Register(&config.Option{
 		Name:            "Prevent Bypassing",
 		Key:             CfgOptionPreventBypassingKey,

profile/endpoints/reason.go

@@ -23,7 +23,7 @@ type reason struct {
 func (r *reason) String() string {
 	prefix := "endpoint in blocklist: "
 	if r.Permitted {
-		prefix = "endpoint in whitelist: "
+		prefix = "endpoint in allowlist: "
 	}
 
 	return prefix + r.description + " " + r.Value

profile/profile-layered.go

@@ -45,6 +45,7 @@ type LayeredProfile struct {
 	FilterSubDomains    config.BoolOption
 	FilterCNAMEs        config.BoolOption
 	PreventBypassing    config.BoolOption
+	DomainHeuristics    config.BoolOption
 }
 
 // NewLayeredProfile returns a new layered profile based on the given local profile.
@@ -108,6 +109,10 @@ func NewLayeredProfile(localProfile *Profile) *LayeredProfile {
 		CfgOptionPreventBypassingKey,
 		cfgOptionPreventBypassing,
 	)
+	new.DomainHeuristics = new.wrapSecurityLevelOption(
+		CfgOptionDomainHeuristicsKey,
+		cfgOptionDomainHeuristics,
+	)
 
 	// TODO: load linked profiles.