From 93367b64df33ce58d74c3af63947b518c47ad88d Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Fri, 29 Jul 2022 16:48:53 +0200 Subject: [PATCH] Bug fixes and performence fixes: Synchronization bug fixed map reseting performence improvment Added more notificationa and better wording better scheduling of file parsing task --- firewall/master.go | 20 ++++++------- intel/customlists/config.go | 3 +- intel/customlists/lists.go | 57 +++++++++++++++++++++++++++++++++---- intel/customlists/module.go | 49 ++++++++++++++++++++++--------- 4 files changed, 99 insertions(+), 30 deletions(-) diff --git a/firewall/master.go b/firewall/master.go index ab368cf3..50d64bc0 100644 --- a/firewall/master.go +++ b/firewall/master.go @@ -51,11 +51,11 @@ var defaultDeciders = []deciderFn{ checkResolverScope, checkConnectivityDomain, checkBypassPrevention, + checkCustomFilterList, checkFilterLists, dropInbound, checkDomainHeuristics, checkAutoPermitRelated, - checkCustomFilterList, } // DecideOnConnection makes a decision about a connection. @@ -618,17 +618,17 @@ matchLoop: func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profile.LayeredProfile, _ packet.Packet) bool { // block if the domain name appears in the custom filter list (check for subdomains if enabled) if conn.Entity.Domain != "" { - if customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()) { - conn.Block("Domains appears in the custom user list", customlists.CfgOptionCustomListBlockingKey) + if ok, match := customlists.LookupDomain(conn.Entity.Domain, p.FilterSubDomains()); ok { + conn.Deny(fmt.Sprintf("domain %s matched %s in custom filter list", conn.Entity.Domain, match), customlists.CfgOptionCustomListBlockingKey) return true } } // block if any of the CNAME appears in the custom filter list (check for subdomains if enabled) - if len(conn.Entity.CNAME) > 0 && p.FilterCNAMEs() { + if p.FilterCNAMEs() { for _, cname := range conn.Entity.CNAME { - if customlists.LookupDomain(cname, p.FilterSubDomains()) { - conn.Block("CNAME appears in the custom user list", customlists.CfgOptionCustomListBlockingKey) + if ok, match := customlists.LookupDomain(cname, p.FilterSubDomains()); ok { + conn.Deny(fmt.Sprintf("domain alias (CNAME) %s matched %s in custom filter list", cname, match), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -636,8 +636,8 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if ip addresses appears in the custom filter list if conn.Entity.IP != nil { - if customlists.LookupIP(&conn.Entity.IP) { - conn.Block("IP appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + if customlists.LookupIP(conn.Entity.IP) { + conn.Deny(fmt.Sprintf("IP address %s appears in the custom filter list", conn.Entity.IP), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -645,7 +645,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block autonomous system by its number if it appears in the custom filter list if conn.Entity.ASN != 0 { if customlists.LookupASN(conn.Entity.ASN) { - conn.Block("ASN appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("autonomous system with number %d appears in the custom filter list", conn.Entity.ASN), customlists.CfgOptionCustomListBlockingKey) return true } } @@ -653,7 +653,7 @@ func checkCustomFilterList(_ context.Context, conn *network.Connection, p *profi // block if the country appears in the custom filter list if conn.Entity.Country != "" { if customlists.LookupCountry(conn.Entity.Country) { - conn.Block("Country appears in the custom filter list", customlists.CfgOptionCustomListBlockingKey) + conn.Deny(fmt.Sprintf("country code %s appears in the custom filter list", conn.Entity.Country), customlists.CfgOptionCustomListBlockingKey) return true } } diff --git a/intel/customlists/config.go b/intel/customlists/config.go index 9b8e0ba7..33e285a9 100644 --- a/intel/customlists/config.go +++ b/intel/customlists/config.go @@ -16,7 +16,7 @@ var ( ) func registerConfig() error { - help := `Put all domains, Ip addresses, country codes and autonomous system that you want to block in a file in where each entry is on a new line. + help := `File that contains list of all domains, Ip addresses, country codes and autonomous system that you want to block, where each entry is on a new line. Lines that start with a '#' symbol are ignored. Everything after the first space/tab is ignored. Example: @@ -53,6 +53,7 @@ AS123 Annotations: config.Annotations{ config.DisplayOrderAnnotation: cfgOptionCustomListBlockingOrder, config.CategoryAnnotation: cfgOptionCustomListCategoryAnnotation, + config.DisplayHintAnnotation: config.DisplayHintFilePicker, }, }) if err != nil { diff --git a/intel/customlists/lists.go b/intel/customlists/lists.go index ad066d0d..e9cbb90a 100644 --- a/intel/customlists/lists.go +++ b/intel/customlists/lists.go @@ -2,6 +2,7 @@ package customlists import ( "bufio" + "fmt" "net" "os" "strconv" @@ -20,14 +21,33 @@ var ( domainsFilterList map[string]struct{} ) -const numberOfZeroIPsUntilWarning = 100 +const ( + numberOfZeroIPsUntilWarning = 100 + customFilterListStatusNotificationID = "intel/customlists_status" + customFilterListZeroIPNotificationID = "intel/customlists_zeroip" +) -func parseFile(filePath string) error { - // reset all maps, previous (if any) settings will be lost +func initFilterLists() { countryCodesFilterList = make(map[string]struct{}) ipAddressesFilterList = make(map[string]struct{}) autonomousSystemsFilterList = make(map[uint]struct{}) domainsFilterList = make(map[string]struct{}) +} + +func parseFile(filePath string) error { + // reset all maps, previous (if any) settings will be lost + for key := range countryCodesFilterList { + delete(countryCodesFilterList, key) + } + for key := range ipAddressesFilterList { + delete(ipAddressesFilterList, key) + } + for key := range autonomousSystemsFilterList { + delete(autonomousSystemsFilterList, key) + } + for key := range domainsFilterList { + delete(domainsFilterList, key) + } // ignore empty file path if filePath == "" { @@ -37,7 +57,21 @@ func parseFile(filePath string) error { // open the file if possible file, err := os.Open(filePath) if err != nil { - log.Warningf("intel/customlists: failed to parse file: \"%s\"", filePath) + log.Warningf("intel/customlists: failed to parse file %q ", err) + // notifications.NotifyWarn("intel/customlists parse failed", "Failed to open custom filter list") + notifications.Notify(¬ifications.Notification{ + EventID: customFilterListStatusNotificationID, + Type: notifications.Warning, + Title: "Failed to open custom filter list", + Message: err.Error(), + ShowOnSystem: false, + AvailableActions: []*notifications.Action{ + { + ID: "ack", + Text: "OK", + }, + }, + }) return err } defer file.Close() @@ -58,11 +92,24 @@ func parseFile(filePath string) error { if numberOfZeroIPs >= numberOfZeroIPsUntilWarning { log.Warning("intel/customlists: Too many zero IP addresses.") - notifications.NotifyWarn("too_many_zero_ips", "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") + notifications.NotifyWarn(customFilterListZeroIPNotificationID, "Too many zero IP addresses. Check your custom filter list.", "Hosts file format is not spported.") } log.Infof("intel/customlists: list loaded successful: %s", filePath) + notifications.NotifyInfo(customFilterListStatusNotificationID, + "Custom filter list loaded successfully.", + fmt.Sprintf(`Custom filter list loaded successfully from file %s +%d domains +%d IPs +%d autonomous systems +%d countries`, + filePath, + len(domainsFilterList), + len(ipAddressesFilterList), + len(autonomousSystemsFilterList), + len(domainsFilterList))) + return nil } diff --git a/intel/customlists/module.go b/intel/customlists/module.go index 15029358..084d1b03 100644 --- a/intel/customlists/module.go +++ b/intel/customlists/module.go @@ -30,7 +30,8 @@ var ( filterListFilePath string filterListFileModifiedTime time.Time - parseLock sync.RWMutex + filterListLock sync.RWMutex + parserTask *modules.Task ) func init() { @@ -38,6 +39,8 @@ func init() { } func prep() error { + initFilterLists() + // register the config in the ui err := registerConfig() if err != nil { @@ -61,23 +64,29 @@ func start() error { return err } - // register timer to run every periodically and check for file updates - module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error { + // create parser task and enqueue for execution. "checkAndUpdateFilterList" will schedule the next execution + parserTask = module.NewTask("intel/customlists file update check", func(context.Context, *modules.Task) error { _ = checkAndUpdateFilterList() return nil - }).Repeat(10 * time.Minute) + }).Schedule(time.Now().Add(20 * time.Second)) - // parse the file at startup - _ = parseFile(getFilePath()) return nil } func checkAndUpdateFilterList() error { - parseLock.Lock() - defer parseLock.Unlock() + filterListLock.Lock() + defer filterListLock.Unlock() - // get path and try to get its info + // get path and ignore if empty filePath := getFilePath() + if filePath == "" { + return nil + } + + // schedule next update check + parserTask.Schedule(time.Now().Add(1 * time.Minute)) + + // try to get file info modifiedTime := time.Now() if fileInfo, err := os.Stat(filePath); err == nil { modifiedTime = fileInfo.ModTime() @@ -96,38 +105,50 @@ func checkAndUpdateFilterList() error { } // LookupIP checks if the IP address is in a custom filter list. -func LookupIP(ip *net.IP) bool { +func LookupIP(ip net.IP) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + _, ok := ipAddressesFilterList[ip.String()] return ok } // LookupDomain checks if the Domain is in a custom filter list. -func LookupDomain(fullDomain string, filterSubdomains bool) bool { +func LookupDomain(fullDomain string, filterSubdomains bool) (bool, string) { + filterListLock.RLock() + defer filterListLock.RUnlock() + if filterSubdomains { // check if domain is in the list and all its subdomains listOfDomains := splitDomain(fullDomain) for _, domain := range listOfDomains { _, ok := domainsFilterList[domain] if ok { - return true + return true, domain } } } else { // check only if the domain is in the list _, ok := domainsFilterList[fullDomain] - return ok + return ok, fullDomain } - return false + return false, "" } // LookupASN checks if the Autonomous system number is in a custom filter list. func LookupASN(number uint) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + _, ok := autonomousSystemsFilterList[number] return ok } // LookupCountry checks if the country code is in a custom filter list. func LookupCountry(countryCode string) bool { + filterListLock.RLock() + defer filterListLock.RUnlock() + _, ok := countryCodesFilterList[countryCode] return ok }