safing-portmaster/service/intel/customlists/lists.go

package customlists

import (
	"bufio"
	"fmt"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/miekg/dns"

	"github.com/safing/portbase/log"
	"github.com/safing/portbase/notifications"
	"github.com/safing/portmaster/service/network/netutils"
)

var (
	countryCodesFilterList      map[string]struct{}
	ipAddressesFilterList       map[string]struct{}
	autonomousSystemsFilterList map[uint]struct{}
	domainsFilterList           map[string]struct{}
)

const (
	rationForInvalidLinesUntilWarning = 0.1
	parseStatusNotificationID         = "customlists:parse-status"
	parseWarningNotificationID        = "customlists:parse-warning"
	zeroIPNotificationID              = "customlists:too-many-zero-ips"
)

func initFilterLists() {
	countryCodesFilterList = make(map[string]struct{})
	ipAddressesFilterList = make(map[string]struct{})
	autonomousSystemsFilterList = make(map[uint]struct{})
	domainsFilterList = make(map[string]struct{})
}

// IsLoaded returns whether a custom filter list is loaded.
func IsLoaded() bool {
	filterListLock.RLock()
	defer filterListLock.RUnlock()

	switch {
	case len(domainsFilterList) > 0:
		return true
	case len(ipAddressesFilterList) > 0:
		return true
	case len(countryCodesFilterList) > 0:
		return true
	case len(autonomousSystemsFilterList) > 0:
		return true
	default:
		return false
	}
}

func parseFile(filePath string) error {
	// Reset all maps, previous (if any) settings will be lost.
	for key := range countryCodesFilterList {
		delete(countryCodesFilterList, key)
	}
	for key := range ipAddressesFilterList {
		delete(ipAddressesFilterList, key)
	}
	for key := range autonomousSystemsFilterList {
		delete(autonomousSystemsFilterList, key)
	}
	for key := range domainsFilterList {
		delete(domainsFilterList, key)
	}

	// Ignore empty file path.
	if filePath == "" {
		return nil
	}

	// Open the file if possible
	file, err := os.Open(filePath)
	if err != nil {
		log.Warningf("intel/customlists: failed to parse file %s", err)
		module.Warning(parseWarningNotificationID, "Failed to open custom filter list", err.Error())
		return err
	}
	defer func() { _ = file.Close() }()

	var allLinesCount uint64
	var invalidLinesCount uint64

	// Read filter file line by line.
	scanner := bufio.NewScanner(file)
	// The scanner will error out if the line is greater than 64K, in this case it is enough.
	for scanner.Scan() {
		allLinesCount++
		// Parse and count invalid lines (comment, empty lines, zero IPs...)
		if !parseLine(scanner.Text()) {
			invalidLinesCount++
		}
	}

	// Check for scanner error.
	if err := scanner.Err(); err != nil {
		return err
	}

	invalidLinesRation := float32(invalidLinesCount) / float32(allLinesCount)

	if invalidLinesRation > rationForInvalidLinesUntilWarning {
		log.Warning("intel/customlists: Too many invalid lines")
		module.Warning(zeroIPNotificationID, "Custom filter list has many invalid lines",
			fmt.Sprintf(`%d out of %d lines are invalid.
			 Check if you are using the correct file format and if the path to the custom filter list is correct.`, invalidLinesCount, allLinesCount))
	} else {
		module.Resolve(zeroIPNotificationID)
	}

	allEntriesCount := len(domainsFilterList) + len(ipAddressesFilterList) + len(autonomousSystemsFilterList) + len(countryCodesFilterList)
	log.Infof("intel/customlists: loaded %d entries from %s", allEntriesCount, filePath)

	notifications.NotifyInfo(parseStatusNotificationID,
		"Custom filter list loaded successfully.",
		fmt.Sprintf(`Custom filter list loaded from file %s:  
%d Domains  
%d IPs  
%d Autonomous Systems  
%d Countries`,
			filePath,
			len(domainsFilterList),
			len(ipAddressesFilterList),
			len(autonomousSystemsFilterList),
			len(countryCodesFilterList)))

	module.Resolve(parseWarningNotificationID)

	return nil
}

func parseLine(line string) (valid bool) {
	// Everything after the first field will be ignored.
	fields := strings.Fields(line)

	// Ignore empty lines.
	if len(fields) == 0 {
		return true // Not an entry, but a valid line.
	}

	field := fields[0]

	// Ignore comments
	if strings.HasPrefix(field, "#") {
		return true // Not an entry, but a valid line.
	}

	// Go through all possible field types.
	// Parsing is ordered by
	// 1. Parsing options (ie. the domain has most variation and goes last.)
	// 2. Speed

	// Check if it'a a country code.
	if isCountryCode(field) {
		countryCodesFilterList[field] = struct{}{}
		return true
	}

	// Check if it's a Autonomous system (example AS123).
	if isAutonomousSystem(field) {
		asNumber, err := strconv.ParseUint(field[2:], 10, 32)
		if err != nil {
			return false
		}
		autonomousSystemsFilterList[uint(asNumber)] = struct{}{}
		return true
	}

	// Try to parse IP address.
	ip := net.ParseIP(field)
	if ip != nil {
		// Check for zero ip.
		if net.IP.Equal(ip, net.IPv4zero) || net.IP.Equal(ip, net.IPv6zero) {
			return false
		}

		ipAddressesFilterList[ip.String()] = struct{}{}
		return true
	}

	// Check if it's a domain.
	domain := dns.Fqdn(field)
	if netutils.IsValidFqdn(domain) {
		domainsFilterList[domain] = struct{}{}
		return true
	}

	return false
}