Change worning for invalid lines in filter list

intel/customlists/config.go

@@ -14,7 +14,7 @@ var (
 var getFilePath config.StringOption
 
 func registerConfig() error {
-	help := `File that contains list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line.  
+	help := `The file should contain list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line.  
 Lines that start with a '#' symbol are ignored.  
 Everything after the first space/tab is ignored.  
 Example:  

intel/customlists/lists.go

@@ -23,9 +23,9 @@ var (
 )
 
 const (
-	numberOfZeroIPsUntilWarning = 100
-	parseStatusNotificationID   = "customlists:parse-status"
-	zeroIPNotificationID        = "customlists:too-many-zero-ips"
+	rationForInvalidLinesUntilWarning = 0.1
+	parseStatusNotificationID         = "customlists:parse-status"
+	zeroIPNotificationID              = "customlists:too-many-zero-ips"
 )
 
 func initFilterLists() {
@@ -62,15 +62,20 @@ func parseFile(filePath string) error {
 		module.Warning(parseStatusNotificationID, "Failed to open custom filter list", err.Error())
 		return err
 	}
-	defer file.Close()
+	defer func() { _ = file.Close() }()
 
-	var numberOfZeroIPs uint64
+	var allLinesCount uint64
+	var invalidLinesCount uint64
 
 	// read filter file line by line.
 	scanner := bufio.NewScanner(file)
 	// the scanner will error out if the line is greater than 64K, in this case it is enough.
 	for scanner.Scan() {
-		parseLine(scanner.Text(), &numberOfZeroIPs)
+		allLinesCount++
+		// parse and count invalid lines (comment, empty lines, zero IPs...)
+		if !parseLine(scanner.Text()) {
+			invalidLinesCount++
+		}
 	}
 
 	// check for scanner error.
@@ -78,9 +83,13 @@ func parseFile(filePath string) error {
 		return err
 	}
 
-	if numberOfZeroIPs >= numberOfZeroIPsUntilWarning {
-		log.Warning("intel/customlists: Too many zero IP addresses.")
-		module.Warning(zeroIPNotificationID, "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.")
+	var invalidLinesRation float32 = float32(invalidLinesCount) / float32(allLinesCount)
+
+	if invalidLinesRation > rationForInvalidLinesUntilWarning {
+		log.Warning("intel/customlists: Too many invalid lines")
+		module.Warning(zeroIPNotificationID, "Check your custom filter list, there is too many invalid lines",
+			fmt.Sprintf(`There are %d from total %d lines that we flagged as invalid.
+			 Check if you are using the correct file format or if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount))
 	} else {
 		module.Resolve(zeroIPNotificationID)
 	}
@@ -105,59 +114,56 @@ func parseFile(filePath string) error {
 	return nil
 }
 
-func parseLine(line string, numberOfZeroIPs *uint64) {
+func parseLine(line string) bool {
 	// everything after the first field will be ignored.
 	fields := strings.Fields(line)
 
 	// ignore empty lines.
 	if len(fields) == 0 {
-		return
+		return false
 	}
 
 	field := fields[0]
 
 	// ignore comments
 	if field[0] == '#' {
-		return
+		return false
 	}
 
 	// check if it'a a country code.
 	if isCountryCode(field) {
 		countryCodesFilterList[field] = struct{}{}
-		return
+		return true
 	}
 
 	// try to parse IP address.
 	ip := net.ParseIP(field)
 	if ip != nil {
-		ipAddressesFilterList[ip.String()] = struct{}{}
-
 		// check for zero ip.
-		if bytes.Compare(ip.To4(), net.IPv4zero) == 0 || bytes.Compare(ip.To16(), net.IPv6zero) == 0 {
-			// check if its zero ip.
-			for i := 0; i < len(ip); i++ {
-				if ip[i] != 0 {
-					*numberOfZeroIPs++
-				}
-			}
+		if bytes.Compare(ip, net.IPv4zero) == 0 || bytes.Compare(ip, net.IPv6zero) == 0 {
+			return false
 		}
-		return
+
+		ipAddressesFilterList[ip.String()] = struct{}{}
+		return true
 	}
 
 	// check if it's a Autonomous system (example AS123).
 	if isAutonomousSystem(field) {
 		asNumber, err := strconv.ParseUint(field[2:], 10, 32)
 		if err != nil {
-			return
+			return false
 		}
 		autonomousSystemsFilterList[uint(asNumber)] = struct{}{}
-		return
+		return true
 	}
 
 	// check if it's a domain.
 	domain := dns.Fqdn(field)
 	if netutils.IsValidFqdn(domain) {
 		domainsFilterList[domain] = struct{}{}
-		return
+		return true
 	}
+
+	return false
 }

intel/customlists/module.go

@@ -41,7 +41,7 @@ func init() {
 func prep() error {
 	initFilterLists()
 
-	// register the config in the ui
+	// register the config in the ui.
 	err := registerConfig()
 	if err != nil {
 		return err
@@ -64,8 +64,8 @@ func start() error {
 		return err
 	}
 
-	// create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution
-	parserTask = module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error {
+	// create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution.
+	parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error {
 		_ = checkAndUpdateFilterList()
 		return nil
 	}).Schedule(time.Now().Add(20 * time.Second))
@@ -119,7 +119,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) {
 	defer filterListLock.RUnlock()
 
 	if filterSubdomains {
-		// check if domain is in the list and all its subdomains
+		// check if domain is in the list and all its subdomains.
 		listOfDomains := splitDomain(fullDomain)
 		for _, domain := range listOfDomains {
 			_, ok := domainsFilterList[domain]