Merge pull request #193 from safing/fix/filterlist-parsing

Update filterlists to use new format

intel/filterlists/cache_version.go

@@ -4,15 +4,20 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/safing/portbase/database"
+
 	"github.com/hashicorp/go-version"
 	"github.com/safing/portbase/database/record"
 )
 
+const resetVersion = "v0.6.0"
+
 type cacheVersionRecord struct {
 	record.Base
 	sync.Mutex
 
 	Version string
+	Reset   string
 }
 
 // getCacheDatabaseVersion reads and returns the cache
@@ -37,6 +42,10 @@ func getCacheDatabaseVersion() (*version.Version, error) {
 		}
 	}
 
+	if verRecord.Reset != resetVersion {
+		return nil, database.ErrNotFound
+	}
+
 	ver, err := version.NewSemver(verRecord.Version)
 	if err != nil {
 		return nil, err
@@ -50,6 +59,7 @@ func getCacheDatabaseVersion() (*version.Version, error) {
 func setCacheDatabaseVersion(ver string) error {
 	verRecord := &cacheVersionRecord{
 		Version: ver,
+		Reset:   resetVersion,
 	}
 
 	verRecord.SetKey(filterListCacheVersionKey)

intel/filterlists/database.go

@@ -200,14 +200,14 @@ func processEntry(ctx context.Context, filter *scopedBloom, entry *listEntry, re
 	normalizeEntry(entry)
 
 	// Only add the entry to the bloom filter if it has any sources.
-	if len(entry.Sources) > 0 {
+	if len(entry.Resources) > 0 {
 		filter.add(entry.Type, entry.Entity)
 	}
 
 	r := &entityRecord{
 		Value:     entry.Entity,
 		Type:      entry.Type,
-		Sources:   entry.Sources,
+		Sources:   entry.getSources(),
 		UpdatedAt: time.Now().Unix(),
 	}
 

intel/filterlists/decoder.go

@@ -8,13 +8,31 @@ import (
 	"io"
 
 	"github.com/safing/portbase/formats/dsd"
+	"github.com/safing/portbase/utils"
 )
 
 type listEntry struct {
-	Entity    string   `json:"entity"`
-	Sources   []string `json:"sources"`
-	Whitelist bool     `json:"whitelist"`
-	Type      string   `json:"type"`
+	Type      string          `json:"type"`
+	Entity    string          `json:"entity"`
+	Whitelist bool            `json:"whitelist"`
+	Resources []entryResource `json:"resources"`
+}
+
+type entryResource struct {
+	SourceID   string `json:"sourceID"`
+	ResourceID string `json:"resourceID"`
+}
+
+func (entry *listEntry) getSources() (sourceIDs []string) {
+	sourceIDs = make([]string, 0, len(entry.Resources))
+
+	for _, resource := range entry.Resources {
+		if !utils.StringInSlice(sourceIDs, resource.SourceID) {
+			sourceIDs = append(sourceIDs, resource.SourceID)
+		}
+	}
+
+	return
 }
 
 // decodeFile decodes a DSDL filterlists file and sends decoded entities to

profile/config-update.go

@@ -4,9 +4,10 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"time"
 
 	"github.com/safing/portbase/config"
-
+	"github.com/safing/portbase/modules"
 	"github.com/safing/portmaster/intel/filterlists"
 	"github.com/safing/portmaster/profile/endpoints"
 )
@@ -25,11 +26,15 @@ func registerConfigUpdater() error {
 		"config",
 		"config change",
 		"update global config profile",
-		updateGlobalConfigProfile,
+		func(ctx context.Context, _ interface{}) error {
+			return updateGlobalConfigProfile(ctx, nil)
+		},
 	)
 }
 
-func updateGlobalConfigProfile(ctx context.Context, data interface{}) error {
+const globalConfigProfileErrorID = "profile:global-profile-error"
+
+func updateGlobalConfigProfile(ctx context.Context, task *modules.Task) error {
 	cfgLock.Lock()
 	defer cfgLock.Unlock()
 
@@ -100,5 +105,27 @@ func updateGlobalConfigProfile(ctx context.Context, data interface{}) error {
 		lastErr = err
 	}
 
+	// If there was any error, try again later until it succeeds.
+	if lastErr == nil {
+		module.Resolve(globalConfigProfileErrorID)
+	} else {
+		// Create task after first failure.
+		if task == nil {
+			task = module.NewTask(
+				"retry updating global config profile",
+				updateGlobalConfigProfile,
+			)
+		}
+
+		// Schedule task.
+		task.Schedule(time.Now().Add(15 * time.Second))
+
+		// Add module warning to inform user.
+		module.Warning(
+			globalConfigProfileErrorID,
+			fmt.Sprintf("Failed to process global settings: %s", err),
+		)
+	}
+
 	return lastErr
 }