Bug fixes and performence fixes:

Synchronization bug fixed
map reseting performence improvment
Added more notificationa and better wording
better scheduling of file parsing task

firewall/master.go

@@ -51,11 +51,11 @@ var defaultDeciders = []deciderFn{
 	checkResolverScope,
 	checkConnectivityDomain,
 	checkBypassPrevention,
+	checkCustomFilterList,
 	checkFilterLists,
 	dropInbound,
 	checkDomainHeuristics,
 	checkAutoPermitRelated,
-	checkCustomFilterList,
 }
 
 // DecideOnConnection makes a decision about a connection.
@@ -618,17 +618,17 @@ matchLoop:
 func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool {
 	// block if the domain name appears in the custom filter list (check for subdomains if enabled)
 	if conn.Entity.Domain != "" {
-		if customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()) {
+		if ok, match := customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()); ok {
-			conn.Block("Domains appears in the custom user list", customlists.CfgOptionCustomListBlockingKey)
+			conn.Deny(fmt.Sprintf("domain %s matched %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey)
 			return true
 		}
 	}
 
 	// block if any of the CNAME appears in the custom filter list (check for subdomains if enabled)
-	if len(conn.Entity.CNAME) > 0 && p.FilterCNAMEs() {
+	if p.FilterCNAMEs() {
 		for _, cname := range conn.Entity.CNAME {
-			if customlists.LookupDomain(cname, p.FilterSubDomains()) {
+			if ok, match := customlists.LookupDomain(cname, p.FilterSubDomains()); ok {
-				conn.Block("CNAME appears in the custom user list", customlists.CfgOptionCustomListBlockingKey)
+				conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matched %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey)
 				return true
 			}
 		}
@@ -636,8 +636,8 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi
 
 	// block if ip addresses appears in the custom filter list
 	if conn.Entity.IP != nil {
-		if customlists.LookupIP(&conn.Entity.IP) {
+		if customlists.LookupIP(conn.Entity.IP) {
-			conn.Block("IP appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey)
+			conn.Deny(fmt.Sprintf("IP address %s appears in the custom filter list", conn.Entity.IP), customlists.CfgOptionCustomListBlockingKey)
 			return true
 		}
 	}
@@ -645,7 +645,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi
 	// block autonomous system by its number if it appears in the custom filter list
 	if conn.Entity.ASN != 0 {
 		if customlists.LookupASN(conn.Entity.ASN) {
-			conn.Block("ASN appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey)
+			conn.Deny(fmt.Sprintf("autonomous system with number %d appears in the custom filter list", conn.Entity.ASN), customlists.CfgOptionCustomListBlockingKey)
 			return true
 		}
 	}
@@ -653,7 +653,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi
 	// block if the country appears in the custom filter list
 	if conn.Entity.Country != "" {
 		if customlists.LookupCountry(conn.Entity.Country) {
-			conn.Block("Country appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey)
+			conn.Deny(fmt.Sprintf("country code %s appears in the custom filter list", conn.Entity.Country), customlists.CfgOptionCustomListBlockingKey)
 			return true
 		}
 	}

intel/customlists/config.go

@@ -16,7 +16,7 @@ var (
 )
 
 func registerConfig() error {
-	help := `Put all domains, Ip addresses, country codes and autonomous system that you want to block in a file in where each entry is on a new line.  
+	help := `File that contains list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line.  
 Lines that start with a '#' symbol are ignored.  
 Everything after the first space/tab is ignored.  
 Example:  
@@ -53,6 +53,7 @@ AS123
 		Annotations: config.Annotations{
 			config.DisplayOrderAnnotation: cfgOptionCustomListBlockingOrder,
 			config.CategoryAnnotation:     cfgOptionCustomListCategoryAnnotation,
+			config.DisplayHintAnnotation:  config.DisplayHintFilePicker,
 		},
 	})
 	if err != nil {

intel/customlists/lists.go

@@ -2,6 +2,7 @@ package customlists
 
 import (
 	"bufio"
+	"fmt"
 	"net"
 	"os"
 	"strconv"
@@ -20,14 +21,33 @@ var (
 	domainsFilterList           map[string]struct{}
 )
 
-const numberOfZeroIPsUntilWarning = 100
+const (
+	numberOfZeroIPsUntilWarning          = 100
+	customFilterListStatusNotificationID = "intel/customlists_status"
+	customFilterListZeroIPNotificationID = "intel/customlists_zeroip"
+)
 
-func parseFile(filePath string) error {
+func initFilterLists() {
-	// reset all maps, previous (if any) settings will be lost
 	countryCodesFilterList = make(map[string]struct{})
 	ipAddressesFilterList = make(map[string]struct{})
 	autonomousSystemsFilterList = make(map[uint]struct{})
 	domainsFilterList = make(map[string]struct{})
+}
+
+func parseFile(filePath string) error {
+	// reset all maps, previous (if any) settings will be lost
+	for key := range countryCodesFilterList {
+		delete(countryCodesFilterList, key)
+	}
+	for key := range ipAddressesFilterList {
+		delete(ipAddressesFilterList, key)
+	}
+	for key := range autonomousSystemsFilterList {
+		delete(autonomousSystemsFilterList, key)
+	}
+	for key := range domainsFilterList {
+		delete(domainsFilterList, key)
+	}
 
 	// ignore empty file path
 	if filePath == "" {
@@ -37,7 +57,21 @@ func parseFile(filePath string) error {
 	// open the file if possible
 	file, err := os.Open(filePath)
 	if err != nil {
-		log.Warningf("intel/customlists: failed to parse file: \"%s\"", filePath)
+		log.Warningf("intel/customlists: failed to parse file %q ", err)
+		// notifications.NotifyWarn("intel/customlists parse failed", "Failed to open custom filter list")
+		notifications.Notify(&notifications.Notification{
+			EventID:      customFilterListStatusNotificationID,
+			Type:         notifications.Warning,
+			Title:        "Failed to open custom filter list",
+			Message:      err.Error(),
+			ShowOnSystem: false,
+			AvailableActions: []*notifications.Action{
+				{
+					ID:   "ack",
+					Text: "OK",
+				},
+			},
+		})
 		return err
 	}
 	defer file.Close()
@@ -58,11 +92,24 @@ func parseFile(filePath string) error {
 
 	if numberOfZeroIPs >= numberOfZeroIPsUntilWarning {
 		log.Warning("intel/customlists: Too many zero IP addresses.")
-		notifications.NotifyWarn("too_many_zero_ips", "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.")
+		notifications.NotifyWarn(customFilterListZeroIPNotificationID, "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.")
 	}
 
 	log.Infof("intel/customlists: list loaded successful: %s", filePath)
 
+	notifications.NotifyInfo(customFilterListStatusNotificationID,
+		"Custom filter list loaded successfully.",
+		fmt.Sprintf(`Custom filter list loaded successfully from file %s  
+%d domains  
+%d IPs  
+%d autonomous systems  
+%d countries`,
+			filePath,
+			len(domainsFilterList),
+			len(ipAddressesFilterList),
+			len(autonomousSystemsFilterList),
+			len(domainsFilterList)))
+
 	return nil
 }
 

intel/customlists/module.go

@@ -30,7 +30,8 @@ var (
 	filterListFilePath         string
 	filterListFileModifiedTime time.Time
 
-	parseLock sync.RWMutex
+	filterListLock sync.RWMutex
+	parserTask     *modules.Task
 )
 
 func init() {
@@ -38,6 +39,8 @@ func init() {
 }
 
 func prep() error {
+	initFilterLists()
+
 	// register the config in the ui
 	err := registerConfig()
 	if err != nil {
@@ -61,23 +64,29 @@ func start() error {
 		return err
 	}
 
-	// register timer to run every periodically and check for file updates
+	// create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution
-	module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error {
+	parserTask = module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error {
 		_ = checkAndUpdateFilterList()
 		return nil
-	}).Repeat(10 * time.Minute)
+	}).Schedule(time.Now().Add(20 * time.Second))
 
-	// parse the file at startup
-	_ = parseFile(getFilePath())
 	return nil
 }
 
 func checkAndUpdateFilterList() error {
-	parseLock.Lock()
+	filterListLock.Lock()
-	defer parseLock.Unlock()
+	defer filterListLock.Unlock()
 
-	// get path and try to get its info
+	// get path and ignore if empty
 	filePath := getFilePath()
+	if filePath == "" {
+		return nil
+	}
+
+	// schedule next update check
+	parserTask.Schedule(time.Now().Add(1 * time.Minute))
+
+	// try to get file info
 	modifiedTime := time.Now()
 	if fileInfo, err := os.Stat(filePath); err == nil {
 		modifiedTime = fileInfo.ModTime()
@@ -96,38 +105,50 @@ func checkAndUpdateFilterList() error {
 }
 
 // LookupIP checks if the IP address is in a custom filter list.
-func LookupIP(ip *net.IP) bool {
+func LookupIP(ip net.IP) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
 	_, ok := ipAddressesFilterList[ip.String()]
 	return ok
 }
 
 // LookupDomain checks if the Domain is in a custom filter list.
-func LookupDomain(fullDomain string, filterSubdomains bool) bool {
+func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
 	if filterSubdomains {
 		// check if domain is in the list and all its subdomains
 		listOfDomains := splitDomain(fullDomain)
 		for _, domain := range listOfDomains {
 			_, ok := domainsFilterList[domain]
 			if ok {
-				return true
+				return true, domain
 			}
 		}
 	} else {
 		// check only if the domain is in the list
 		_, ok := domainsFilterList[fullDomain]
-		return ok
+		return ok, fullDomain
 	}
-	return false
+	return false, ""
 }
 
 // LookupASN checks if the Autonomous system number is in a custom filter list.
 func LookupASN(number uint) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
 	_, ok := autonomousSystemsFilterList[number]
 	return ok
 }
 
 // LookupCountry checks if the country code is in a custom filter list.
 func LookupCountry(countryCode string) bool {
+	filterListLock.RLock()
+	defer filterListLock.RUnlock()
+
 	_, ok := countryCodesFilterList[countryCode]
 	return ok
 }