From ced6690acdf77dabbe922b90b1f1f910e719073c Mon Sep 17 00:00:00 2001 From: Daniel Date: Mon, 1 Aug 2022 14:11:01 +0200 Subject: [PATCH] Minor improvements and documentation --- intel/customlists/config.go | 2 +- intel/customlists/lists.go | 67 ++++++++++++++++++++----------------- intel/customlists/module.go | 20 +++++------ 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/intel/customlists/config.go b/intel/customlists/config.go index a50c4255..f58f2f67 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -28,7 +28,7 @@ Lists in the "Hosts" format are not supported. Please note that the custom filter list is fully loaded into memory. This can have a negative impact on your device if big lists are loaded.` - // register a setting for the file path in the ui + // Register a setting for the file path in the ui err := config.Register(&config.Option{ Name: "Custom Filter List", Key: CfgOptionCustomListBlockingKey, diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index 51907f06..81bb8929 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -37,7 +37,7 @@ func initFilterLists() { } func parseFile(filePath string) error { - // reset all maps, previous (if any) settings will be lost. + // Reset all maps, previous (if any) settings will be lost. for key := range countryCodesFilterList { delete(countryCodesFilterList, key) } @@ -51,12 +51,12 @@ func parseFile(filePath string) error { delete(domainsFilterList, key) } - // ignore empty file path. + // Ignore empty file path. if filePath == "" { return nil } - // open the file if possible + // Open the file if possible file, err := os.Open(filePath) if err != nil { log.Warningf("intel/customlists: failed to parse file %s", err) @@ -68,18 +68,18 @@ func parseFile(filePath string) error { var allLinesCount uint64 var invalidLinesCount uint64 - // read filter file line by line. + // Read filter file line by line. scanner := bufio.NewScanner(file) - // the scanner will error out if the line is greater than 64K, in this case it is enough. + // The scanner will error out if the line is greater than 64K, in this case it is enough. for scanner.Scan() { allLinesCount++ - // parse and count invalid lines (comment, empty lines, zero IPs...) + // Parse and count invalid lines (comment, empty lines, zero IPs...) if !parseLine(scanner.Text()) { invalidLinesCount++ } } - // check for scanner error. + // Check for scanner error. if err := scanner.Err(); err != nil { return err } @@ -88,8 +88,8 @@ func parseFile(filePath string) error { if invalidLinesRation > rationForInvalidLinesUntilWarning { log.Warning("intel/customlists: Too many invalid lines") - module.Warning(zeroIPNotificationID, "Custom filter list has many invalid entries", - fmt.Sprintf(`%d out of %d entires are invalid. + module.Warning(zeroIPNotificationID, "Custom filter list has many invalid lines", + fmt.Sprintf(`%d out of %d lines are invalid. Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount)) } else { module.Resolve(zeroIPNotificationID) @@ -116,41 +116,34 @@ func parseFile(filePath string) error { return nil } -func parseLine(line string) bool { - // everything after the first field will be ignored. +func parseLine(line string) (valid bool) { + // Everything after the first field will be ignored. fields := strings.Fields(line) - // ignore empty lines. + // Ignore empty lines. if len(fields) == 0 { - return false + return true // Not an entry, but a valid line. } field := fields[0] - // ignore comments - if field[0] == '#' { - return false + // Ignore comments + if strings.HasPrefix(field, "#") { + return true // Not an entry, but a valid line. } - // check if it'a a country code. + // Go through all possible field types. + // Parsing is ordered by + // 1. Parsing options (ie. the domain has most variation and goes last.) + // 2. Speed + + // Check if it'a a country code. if isCountryCode(field) { countryCodesFilterList[field] = struct{}{} return true } - // try to parse IP address. - ip := net.ParseIP(field) - if ip != nil { - // check for zero ip. - if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) { - return false - } - - ipAddressesFilterList[ip.String()] = struct{}{} - return true - } - - // check if it's a Autonomous system (example AS123). + // Check if it's a Autonomous system (example AS123). if isAutonomousSystem(field) { asNumber, err := strconv.ParseUint(field[2:], 10, 32) if err != nil { @@ -160,7 +153,19 @@ func parseLine(line string) bool { return true } - // check if it's a domain. + // Try to parse IP address. + ip := net.ParseIP(field) + if ip != nil { + // Check for zero ip. + if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) { + return false + } + + ipAddressesFilterList[ip.String()] = struct{}{} + return true + } + + // Check if it's a domain. domain := dns.Fqdn(field) if netutils.IsValidFqdn(domain) { domainsFilterList[domain] = struct{}{} diff --git a/intel/customlists/module.go b/intel/customlists/module.go index e1d187e0..e20cc890 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -43,7 +43,7 @@ func init() { func prep() error { initFilterLists() - // register the config in the ui. + // Register the config in the ui. err := registerConfig() if err != nil { return err @@ -53,7 +53,7 @@ func prep() error { } func start() error { - // register to hook to update after config change. + // Register to hook to update after config change. if err := module.RegisterEventHook( configModuleName, configChangeEvent, @@ -66,13 +66,13 @@ func start() error { return err } - // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. + // Create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution. parserTask = module.NewTask("intel/customlists:file-update-check", func(context.Context, *modules.Task) error { checkAndUpdateFilterList() return nil }).Schedule(time.Now().Add(20 * time.Second)) - // register api endpoint for updating the filter list + // Register api endpoint for updating the filter list. if err := api.RegisterEndpoint(api.Endpoint{ Path: "customlists/update", Write: api.PermitUser, @@ -94,22 +94,22 @@ func checkAndUpdateFilterList() { filterListLock.Lock() defer filterListLock.Unlock() - // get path and ignore if empty + // Get path and ignore if empty filePath := getFilePath() if filePath == "" { return } - // schedule next update check + // Schedule next update check parserTask.Schedule(time.Now().Add(1 * time.Minute)) - // try to get file info + // Try to get file info modifiedTime := time.Now() if fileInfo, err := os.Stat(filePath); err == nil { modifiedTime = fileInfo.ModTime() } - // check if file path has changed or if modified time has changed + // Check if file path has changed or if modified time has changed if filterListFilePath != filePath || !filterListFileModifiedTime.Equal(modifiedTime) { err := parseFile(filePath) if err != nil { @@ -135,7 +135,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { defer filterListLock.RUnlock() if filterSubdomains { - // check if domain is in the list and all its subdomains. + // Check if domain is in the list and all its subdomains. listOfDomains := splitDomain(fullDomain) for _, domain := range listOfDomains { _, ok := domainsFilterList[domain] @@ -144,7 +144,7 @@ func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { } } } else { - // check only if the domain is in the list + // Check only if the domain is in the list _, ok := domainsFilterList[fullDomain] return ok, fullDomain }