Add support for signed indexes

updater/fetch.go

@@ -45,7 +45,12 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client,
 		verifOpts    = reg.GetVerificationOptions(rv.resource.Identifier)
 	)
 	if verifOpts != nil {
-		verifiedHash, sigFileData, err = reg.fetchAndVerifySigFile(ctx, client, rv, verifOpts, tries)
+		verifiedHash, sigFileData, err = reg.fetchAndVerifySigFile(
+			ctx, client,
+			verifOpts, rv.versionedPath()+filesig.Extension, rv.SigningMetadata(),
+			tries,
+		)
+
 		if err != nil {
 			switch verifOpts.DownloadPolicy {
 			case SignaturePolicyRequire:
@@ -142,9 +147,9 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client,
 	return nil
 }
 
-func (reg *ResourceRegistry) fetchAndVerifySigFile(ctx context.Context, client *http.Client, rv *ResourceVersion, verifOpts *VerificationOptions, tries int) (*lhash.LabeledHash, []byte, error) {
+func (reg *ResourceRegistry) fetchAndVerifySigFile(ctx context.Context, client *http.Client, verifOpts *VerificationOptions, sigFilePath string, requiredMetadata map[string]string, tries int) (*lhash.LabeledHash, []byte, error) {
 	// Download signature file.
-	resp, _, err := reg.makeRequest(ctx, client, rv.versionedPath()+filesig.Extension, tries)
+	resp, _, err := reg.makeRequest(ctx, client, sigFilePath, tries)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -172,7 +177,7 @@ func (reg *ResourceRegistry) fetchAndVerifySigFile(ctx context.Context, client *
 	for _, sig := range sigs {
 		fd, err := filesig.VerifyFileData(
 			sig,
-			rv.SigningMetadata(),
+			requiredMetadata,
 			verifOpts.TrustStore,
 		)
 		if err != nil {
@@ -192,12 +197,12 @@ func (reg *ResourceRegistry) fetchAndVerifySigFile(ctx context.Context, client *
 	return verifiedHash, sigFileData, nil
 }
 
-func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client, downloadPath string, tries int) ([]byte, error) {
+func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client, downloadPath string, tries int) (fileData []byte, downloadedFrom string, err error) {
 	// backoff when retrying
 	if tries > 0 {
 		select {
 		case <-ctx.Done():
-			return nil, nil // module is shutting down
+			return nil, "", nil // module is shutting down
 		case <-time.After(time.Duration(tries*tries) * time.Second):
 		}
 	}
@@ -205,7 +210,7 @@ func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client,
 	// start file download
 	resp, downloadURL, err := reg.makeRequest(ctx, client, downloadPath, tries)
 	if err != nil {
-		return nil, err
+		return nil, downloadURL, err
 	}
 	defer func() {
 		_ = resp.Body.Close()
@@ -215,13 +220,13 @@ func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client,
 	buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength))
 	n, err := io.Copy(buf, resp.Body)
 	if err != nil {
-		return nil, fmt.Errorf("failed to download %q: %w", downloadURL, err)
+		return nil, downloadURL, fmt.Errorf("failed to download %q: %w", downloadURL, err)
 	}
 	if resp.ContentLength != n {
-		return nil, fmt.Errorf("failed to finish download of %q: written %d out of %d bytes", downloadURL, n, resp.ContentLength)
+		return nil, downloadURL, fmt.Errorf("failed to finish download of %q: written %d out of %d bytes", downloadURL, n, resp.ContentLength)
 	}
 
-	return buf.Bytes(), nil
+	return buf.Bytes(), downloadURL, nil
 }
 
 func (reg *ResourceRegistry) makeRequest(ctx context.Context, client *http.Client, downloadPath string, tries int) (resp *http.Response, downloadURL string, err error) {

updater/get.go

@@ -34,7 +34,7 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) {
 		// Verify file, if configured.
 		_, err := file.Verify()
 		if err != nil && !errors.Is(err, ErrVerificationNotConfigured) {
-			// FIXME: If verification is required, try deleting the resource and downloading it again.
+			// TODO: If verification is required, try deleting the resource and downloading it again.
 			return nil, fmt.Errorf("failed to verify file: %w", err)
 		}
 

updater/indexes.go

@@ -7,6 +7,11 @@ import (
 	"time"
 )
 
+const (
+	baseIndexExtension = ".json"
+	v2IndexExtension   = ".v2.json"
+)
+
 // Index describes an index file pulled by the updater.
 type Index struct {
 	// Path is the path to the index file
@@ -35,21 +40,25 @@ type IndexFile struct {
 }
 
 var (
-	// ErrIndexFromFuture is returned when a signed index is parsed with a
+	// ErrIndexChecksumMismatch is returned when an index does not match its
+	// signed checksum.
+	ErrIndexChecksumMismatch = errors.New("index checksum does mot match signature")
+
+	// ErrIndexFromFuture is returned when an index is parsed with a
 	// Published timestamp that lies in the future.
 	ErrIndexFromFuture = errors.New("index is from the future")
 
-	// ErrIndexIsOlder is returned when a signed index is parsed with an older
+	// ErrIndexIsOlder is returned when an index is parsed with an older
 	// Published timestamp than the current Published timestamp.
 	ErrIndexIsOlder = errors.New("index is older than the current one")
 
-	// ErrIndexChannelMismatch is returned when a signed index is parsed with a
+	// ErrIndexChannelMismatch is returned when an index is parsed with a
 	// different channel that the expected one.
 	ErrIndexChannelMismatch = errors.New("index does not match the expected channel")
 )
 
 // ParseIndexFile parses an index file and checks if it is valid.
-func ParseIndexFile(indexData []byte, channel string, currentPublished time.Time) (*IndexFile, error) {
+func ParseIndexFile(indexData []byte, channel string, lastIndexRelease time.Time) (*IndexFile, error) {
 	// Load into struct.
 	indexFile := &IndexFile{}
 	err := json.Unmarshal(indexData, indexFile)
@@ -69,8 +78,8 @@ func ParseIndexFile(indexData []byte, channel string, currentPublished time.Time
 		return indexFile, ErrIndexFromFuture
 
 	case !indexFile.Published.IsZero() &&
-		!currentPublished.IsZero() &&
+		!lastIndexRelease.IsZero() &&
-		currentPublished.After(indexFile.Published):
+		lastIndexRelease.After(indexFile.Published):
 		return indexFile, ErrIndexIsOlder
 
 	case channel != "" &&

updater/indexes_test.go

@@ -33,6 +33,8 @@ var (
 )
 
 func TestIndexParsing(t *testing.T) {
+	t.Parallel()
+
 	lastRelease, err := time.Parse(time.RFC3339, "2022-01-01T00:00:00Z")
 	if err != nil {
 		t.Fatal(err)

updater/registry.go

@@ -23,7 +23,7 @@ type ResourceRegistry struct {
 	Name       string
 	storageDir *utils.DirStructure
 	tmpDir     *utils.DirStructure
-	indexes    []Index
+	indexes    []*Index
 
 	resources        map[string]*Resource
 	UpdateURLs       []string
@@ -57,7 +57,7 @@ func (reg *ResourceRegistry) AddIndex(idx Index) {
 		filepath.Base(idx.Path), filepath.Ext(idx.Path),
 	)
 
-	reg.indexes = append(reg.indexes, idx)
+	reg.indexes = append(reg.indexes, &idx)
 }
 
 // Initialize initializes a raw registry struct and makes it ready for usage.
@@ -225,7 +225,7 @@ func (reg *ResourceRegistry) ResetIndexes() {
 	reg.Lock()
 	defer reg.Unlock()
 
-	reg.indexes = make([]Index, 0, 5)
+	reg.indexes = make([]*Index, 0, len(reg.indexes))
 }
 
 // Cleanup removes temporary files.

updater/signing.go

@@ -1,12 +1,7 @@
 package updater
 
 import (
-	"errors"
-	"fmt"
 	"strings"
-	"time"
-
-	"github.com/safing/portbase/formats/dsd"
 
 	"github.com/safing/jess"
 )
@@ -52,76 +47,3 @@ const (
 	// SignaturePolicyDisable only downloads signatures, but does not verify them.
 	SignaturePolicyDisable
 )
-
-// IndexFile represents an index file.
-type IndexFile struct {
-	Channel   string
-	Published time.Time
-	Expires   time.Time
-
-	Versions map[string]string
-}
-
-var (
-	// ErrIndexFromFuture is returned when a signed index is parsed with a
-	// Published timestamp that lies in the future.
-	ErrIndexFromFuture = errors.New("index is from the future")
-
-	// ErrIndexIsOlder is returned when a signed index is parsed with an older
-	// Published timestamp than the current Published timestamp.
-	ErrIndexIsOlder = errors.New("index is older than the current one")
-
-	// ErrIndexChannelMismatch is returned when a signed index is parsed with a
-	// different channel that the expected one.
-	ErrIndexChannelMismatch = errors.New("index does not match the expected channel")
-
-	// ErrIndexExpired is returned when a signed index is parsed with a Expires
-	// timestamp in the past.
-	ErrIndexExpired = errors.New("index has expired")
-
-	verificationRequirements = jess.NewRequirements().
-					Remove(jess.Confidentiality).
-					Remove(jess.RecipientAuthentication)
-)
-
-// ParseIndex parses the signed index and checks if it is valid.
-func ParseIndex(indexData []byte, verifOpts *VerificationOptions, channel string, currentPublished time.Time) (*IndexFile, error) {
-	// FIXME: fall back to the old index format.
-	// FIXME: use this function for index parsing.
-
-	// Parse data.
-	letter, err := jess.LetterFromDSD(indexData)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse signed index: %w", err)
-	}
-
-	// Verify signatures.
-	signedIndexData, err := letter.Open(verificationRequirements, verifOpts.TrustStore)
-	if err != nil {
-		return nil, fmt.Errorf("failed to verify signature: %w", err)
-	}
-
-	// Load into struct.
-	signedIndex := &IndexFile{}
-	_, err = dsd.Load(signedIndexData, signedIndex)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse signed index data: %w", err)
-	}
-
-	// Check the index metadata.
-	switch {
-	case time.Now().Before(signedIndex.Published):
-		return signedIndex, ErrIndexFromFuture
-
-	case time.Now().After(signedIndex.Expires):
-		return signedIndex, ErrIndexExpired
-
-	case currentPublished.After(signedIndex.Published):
-		return signedIndex, ErrIndexIsOlder
-
-	case channel != "" && channel != signedIndex.Channel:
-		return signedIndex, ErrIndexChannelMismatch
-	}
-
-	return signedIndex, nil
-}

updater/storage.go

@@ -115,15 +115,18 @@ func (reg *ResourceRegistry) LoadIndexes(ctx context.Context) error {
 	return firstErr
 }
 
-func (reg *ResourceRegistry) getIndexes() []Index {
+// getIndexes returns a copy of the index.
+// The indexes itself are references.
+func (reg *ResourceRegistry) getIndexes() []*Index {
 	reg.RLock()
 	defer reg.RUnlock()
-	indexes := make([]Index, len(reg.indexes))
+
+	indexes := make([]*Index, len(reg.indexes))
 	copy(indexes, reg.indexes)
 	return indexes
 }
 
-func (reg *ResourceRegistry) loadIndexFile(idx Index) error {
+func (reg *ResourceRegistry) loadIndexFile(idx *Index) error {
 	path := filepath.FromSlash(idx.Path)
 	data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, path))
 	if err != nil {
@@ -137,9 +140,6 @@ func (reg *ResourceRegistry) loadIndexFile(idx Index) error {
 	}
 
 	// Update last seen release.
-	// TODO: We are working with a copy here, so this has no effect.
-	// This does not break to current implementation, but make the
-	// protection ineffective.
 	idx.LastRelease = indexFile.Published
 
 	// Warn if there aren't any releases in the index.

updater/updating.go

@@ -10,6 +10,8 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/safing/jess/filesig"
+	"github.com/safing/jess/lhash"
 	"github.com/safing/portbase/log"
 	"github.com/safing/portbase/utils"
 )
@@ -36,23 +38,73 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
 	return nil
 }
 
-func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Client, idx Index) error {
+func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Client, idx *Index) error {
-	var err error
+	var (
-	var data []byte
+		// Index.
+		indexErr    error
+		indexData   []byte
+		downloadURL string
 
-	// download new index
+		// Signature.
-	for tries := 0; tries < 3; tries++ {
+		sigErr       error
-		data, err = reg.fetchData(ctx, client, idx.Path, tries)
+		verifiedHash *lhash.LabeledHash
-		if err == nil {
+		sigFileData  []byte
-			break
+		verifOpts    = reg.GetVerificationOptions(idx.Path)
-		}
+	)
+
+	// Upgrade to v2 index if verification is enabled.
+	downloadIndexPath := idx.Path
+	if verifOpts != nil {
+		downloadIndexPath = strings.TrimSuffix(downloadIndexPath, baseIndexExtension) + v2IndexExtension
 	}
-	if err != nil {
+
-		return fmt.Errorf("failed to download index %s: %w", idx.Path, err)
+	// Download new index and signature.
+	for tries := 0; tries < 3; tries++ {
+		// Index and signature need to be fetched together, so that they are
+		// fetched from the same source. One source should always have a matching
+		// index and signature. Backup sources may be behind a little.
+		// If the signature verification fails, another source should be tried.
+
+		// Get index data.
+		indexData, downloadURL, indexErr = reg.fetchData(ctx, client, downloadIndexPath, tries)
+		if indexErr != nil {
+			log.Debugf("%s: failed to fetch index %s: %s", reg.Name, downloadURL, indexErr)
+			continue
+		}
+
+		// Get signature and verify it.
+		if verifOpts != nil {
+			verifiedHash, sigFileData, sigErr = reg.fetchAndVerifySigFile(
+				ctx, client,
+				verifOpts, downloadIndexPath+filesig.Extension, nil,
+				tries,
+			)
+			if sigErr != nil {
+				log.Debugf("%s: failed to verify signature of %s: %s", reg.Name, downloadURL, sigErr)
+				continue
+			}
+
+			// Check if the index matches the verified hash.
+			if verifiedHash.MatchesData(indexData) {
+				log.Infof("%s: verified signature of %s", reg.Name, downloadURL)
+			} else {
+				sigErr = ErrIndexChecksumMismatch
+				log.Debugf("%s: checksum does not match file from %s", reg.Name, downloadURL)
+				continue
+			}
+		}
+
+		break
+	}
+	if indexErr != nil {
+		return fmt.Errorf("failed to fetch index %s: %w", downloadIndexPath, indexErr)
+	}
+	if sigErr != nil {
+		return fmt.Errorf("failed to fetch or verify index %s signature: %w", downloadIndexPath, sigErr)
 	}
 
 	// Parse the index file.
-	indexFile, err := ParseIndexFile(data, idx.Channel, idx.LastRelease)
+	indexFile, err := ParseIndexFile(indexData, idx.Channel, idx.LastRelease)
 	if err != nil {
 		return fmt.Errorf("failed to parse index %s: %w", idx.Path, err)
 	}
@@ -83,21 +135,33 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli
 		log.Debugf("%s: index %s is empty", reg.Name, idx.Path)
 	}
 
-	// check if dest dir exists
+	// Check if dest dir exists.
 	indexDir := filepath.FromSlash(path.Dir(idx.Path))
 	err = reg.storageDir.EnsureRelPath(indexDir)
 	if err != nil {
 		log.Warningf("%s: failed to ensure directory for updated index %s: %s", reg.Name, idx.Path, err)
 	}
 
-	// save index
-	indexPath := filepath.FromSlash(idx.Path)
 	// Index files must be readable by portmaster-staert with user permissions in order to load the index.
-	err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, indexPath), data, 0o0644) //nolint:gosec
+	err = ioutil.WriteFile( //nolint:gosec
+		filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)),
+		indexData, 0o0644,
+	)
 	if err != nil {
 		log.Warningf("%s: failed to save updated index %s: %s", reg.Name, idx.Path, err)
 	}
 
+	// Write signature file, if we have one.
+	if len(sigFileData) > 0 {
+		err = ioutil.WriteFile( //nolint:gosec
+			filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)+filesig.Extension),
+			sigFileData, 0o0644,
+		)
+		if err != nil {
+			log.Warningf("%s: failed to save updated index signature %s: %s", reg.Name, idx.Path+filesig.Extension, err)
+		}
+	}
+
 	log.Infof("%s: updated index %s with %d entries", reg.Name, idx.Path, len(indexFile.Releases))
 	return nil
 }