Minor improvements and documentation

intel/customlists/config.go

@@ -28,7 +28,7 @@ Lists in the "Hosts" format are not supported.
 
 Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.`
 
-	// register a setting for the file path in the ui
+	// Register a setting for the file path in the ui
 	err := config.Register(&config.Option{
 		Name:            "Custom Filter List",
 		Key:             CfgOptionCustomListBlockingKey,

intel/customlists/lists.go

@@ -37,7 +37,7 @@ func initFilterLists() {
 }
 
 func parseFile(filePath string) error {
-	// reset all maps, previous (if any) settings will be lost.
+	// Reset all maps, previous (if any) settings will be lost.
 	for key := range countryCodesFilterList {
 		delete(countryCodesFilterList, key)
 	}
@@ -51,12 +51,12 @@ func parseFile(filePath string) error {
 		delete(domainsFilterList, key)
 	}
 
-	// ignore empty file path.
+	// Ignore empty file path.
 	if filePath == "" {
 		return nil
 	}
 
-	// open the file if possible
+	// Open the file if possible
 	file, err := os.Open(filePath)
 	if err != nil {
 		log.Warningf("intel/customlists: failed to parse file %s", err)
@@ -68,18 +68,18 @@ func parseFile(filePath string) error {
 	var allLinesCount uint64
 	var invalidLinesCount uint64
 
-	// read filter file line by line.
+	// Read filter file line by line.
 	scanner := bufio.NewScanner(file)
-	// the scanner will error out if the line is greater than 64K, in this case it is enough.
+	// The scanner will error out if the line is greater than 64K, in this case it is enough.
 	for scanner.Scan() {
 		allLinesCount++
-		// parse and count invalid lines (comment, empty lines, zero IPs...)
+		// Parse and count invalid lines (comment, empty lines, zero IPs...)
 		if !parseLine(scanner.Text()) {
 			invalidLinesCount++
 		}
 	}
 
-	// check for scanner error.
+	// Check for scanner error.
 	if err := scanner.Err(); err != nil {
 		return err
 	}
@@ -88,8 +88,8 @@ func parseFile(filePath string) error {
 
 	if invalidLinesRation > rationForInvalidLinesUntilWarning {
 		log.Warning("intel/customlists: Too many invalid lines")
-		module.Warning(zeroIPNotificationID, "Custom filter list has many invalid entries",
+		module.Warning(zeroIPNotificationID, "Custom filter list has many invalid lines",
-			fmt.Sprintf(`%d out of %d entires are invalid.
+			fmt.Sprintf(`%d out of %d lines are invalid.
 			 Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount))
 	} else {
 		module.Resolve(zeroIPNotificationID)
@@ -116,41 +116,34 @@ func parseFile(filePath string) error {
 	return nil
 }
 
-func parseLine(line string) bool {
+func parseLine(line string) (valid bool) {
-	// everything after the first field will be ignored.
+	// Everything after the first field will be ignored.
 	fields := strings.Fields(line)
 
-	// ignore empty lines.
+	// Ignore empty lines.
 	if len(fields) == 0 {
-		return false
+		return true // Not an entry, but a valid line.
 	}
 
 	field := fields[0]
 
-	// ignore comments
+	// Ignore comments
-	if field[0] == '#' {
+	if strings.HasPrefix(field, "#") {
-		return false
+		return true // Not an entry, but a valid line.
 	}
 
-	// check if it'a a country code.
+	// Go through all possible field types.
+	// Parsing is ordered by
+	// 1. Parsing options (ie. the domain has most variation and goes last.)
+	// 2. Speed
+
+	// Check if it'a a country code.
 	if isCountryCode(field) {
 		countryCodesFilterList[field] = struct{}{}
 		return true
 	}
 
-	// try to parse IP address.
+	// Check if it's a Autonomous system (example AS123).
-	ip := net.ParseIP(field)
-	if ip != nil {
-		// check for zero ip.
-		if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) {
-			return false
-		}
-
-		ipAddressesFilterList[ip.String()] = struct{}{}
-		return true
-	}
-
-	// check if it's a Autonomous system (example AS123).
 	if isAutonomousSystem(field) {
 		asNumber, err := strconv.ParseUint(field[2:], 10, 32)
 		if err != nil {
@@ -160,7 +153,19 @@ func parseLine(line string) bool {
 		return true
 	}
 
-	// check if it's a domain.
+	// Try to parse IP address.
+	ip := net.ParseIP(field)
+	if ip != nil {
+		// Check for zero ip.
+		if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) {
+			return false
+		}
+
+		ipAddressesFilterList[ip.String()] = struct{}{}
+		return true
+	}
+
+	// Check if it's a domain.
 	domain := dns.Fqdn(field)
 	if netutils.IsValidFqdn(domain) {
 		domainsFilterList[domain] = struct{}{}

intel/customlists/module.go

@@ -43,7 +43,7 @@ func init() {
 func prep() error {
 	initFilterLists()
 
-	// register the config in the ui.
+	// Register the config in the ui.
 	err := registerConfig()
 	if err != nil {
 		return err
@@ -53,7 +53,7 @@ func prep() error {
 }
 
 func start() error {
-	// register to hook to update after config change.
+	// Register to hook to update after config change.
 	if err := module.RegisterEventHook(
 		configModuleName,
 		configChangeEvent,
@@ -66,13 +66,13 @@ func start() error {
 		return err
 	}
 
-	// create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution.
+	// Create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution.
 	parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error {
 		checkAndUpdateFilterList()
 		return nil
 	}).Schedule(time.Now().Add(20 * time.Second))
 
-	// register api endpoint for updating the filter list
+	// Register api endpoint for updating the filter list.
 	if err := api.RegisterEndpoint(api.Endpoint{
 		Path:      "customlists/update",
 		Write:     api.PermitUser,
@@ -94,22 +94,22 @@ func checkAndUpdateFilterList() {
 	filterListLock.Lock()
 	defer filterListLock.Unlock()
 
-	// get path and ignore if empty
+	// Get path and ignore if empty
 	filePath := getFilePath()
 	if filePath == "" {
 		return
 	}
 
-	// schedule next update check
+	// Schedule next update check
 	parserTask.Schedule(time.Now().Add(1 * time.Minute))
 
-	// try to get file info
+	// Try to get file info
 	modifiedTime := time.Now()
 	if fileInfo, err := os.Stat(filePath); err == nil {
 		modifiedTime = fileInfo.ModTime()
 	}
 
-	// check if file path has changed or if modified time has changed
+	// Check if file path has changed or if modified time has changed
 	if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) {
 		err := parseFile(filePath)
 		if err != nil {
@@ -135,7 +135,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) {
 	defer filterListLock.RUnlock()
 
 	if filterSubdomains {
-		// check if domain is in the list and all its subdomains.
+		// Check if domain is in the list and all its subdomains.
 		listOfDomains := splitDomain(fullDomain)
 		for _, domain := range listOfDomains {
 			_, ok := domainsFilterList[domain]
@@ -144,7 +144,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) {
 			}
 		}
 	} else {
-		// check only if the domain is in the list
+		// Check only if the domain is in the list
 		_, ok := domainsFilterList[fullDomain]
 		return ok, fullDomain
 	}