package updater import ( "context" "fmt" "net/http" "os" "path" "path/filepath" "strings" "sync" "github.com/safing/jess/filesig" "github.com/safing/jess/lhash" "github.com/safing/portbase/log" "github.com/safing/portbase/utils" ) // UpdateIndexes downloads all indexes. An error is only returned when all // indexes fail to update. func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error { var lastErr error var anySuccess bool client := &http.Client{} for _, idx := range reg.getIndexes() { if err := reg.downloadIndex(ctx, client, idx); err != nil { lastErr = err log.Warningf("%s: failed to update index %s: %s", reg.Name, idx.Path, err) } else { anySuccess = true } } if !anySuccess { return fmt.Errorf("failed to update all indexes, last error was: %w", lastErr) } return nil } func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Client, idx *Index) error { var ( // Index. indexErr error indexData []byte downloadURL string // Signature. sigErr error verifiedHash *lhash.LabeledHash sigFileData []byte verifOpts = reg.GetVerificationOptions(idx.Path) ) // Upgrade to v2 index if verification is enabled. downloadIndexPath := idx.Path if verifOpts != nil { downloadIndexPath = strings.TrimSuffix(downloadIndexPath, baseIndexExtension) + v2IndexExtension } // Download new index and signature. for tries := 0; tries < 3; tries++ { // Index and signature need to be fetched together, so that they are // fetched from the same source. One source should always have a matching // index and signature. Backup sources may be behind a little. // If the signature verification fails, another source should be tried. // Get index data. indexData, downloadURL, indexErr = reg.fetchData(ctx, client, downloadIndexPath, tries) if indexErr != nil { log.Debugf("%s: failed to fetch index %s: %s", reg.Name, downloadURL, indexErr) continue } // Get signature and verify it. if verifOpts != nil { verifiedHash, sigFileData, sigErr = reg.fetchAndVerifySigFile( ctx, client, verifOpts, downloadIndexPath+filesig.Extension, nil, tries, ) if sigErr != nil { log.Debugf("%s: failed to verify signature of %s: %s", reg.Name, downloadURL, sigErr) continue } // Check if the index matches the verified hash. if verifiedHash.Matches(indexData) { log.Infof("%s: verified signature of %s", reg.Name, downloadURL) } else { sigErr = ErrIndexChecksumMismatch log.Debugf("%s: checksum does not match file from %s", reg.Name, downloadURL) continue } } break } if indexErr != nil { return fmt.Errorf("failed to fetch index %s: %w", downloadIndexPath, indexErr) } if sigErr != nil { return fmt.Errorf("failed to fetch or verify index %s signature: %w", downloadIndexPath, sigErr) } // Parse the index file. indexFile, err := ParseIndexFile(indexData, idx.Channel, idx.LastRelease) if err != nil { return fmt.Errorf("failed to parse index %s: %w", idx.Path, err) } // Add index data to registry. if len(indexFile.Releases) > 0 { // Check if all resources are within the indexes' authority. authoritativePath := path.Dir(idx.Path) + "/" if authoritativePath == "./" { // Fix path for indexes at the storage root. authoritativePath = "" } cleanedData := make(map[string]string, len(indexFile.Releases)) for key, version := range indexFile.Releases { if strings.HasPrefix(key, authoritativePath) { cleanedData[key] = version } else { log.Warningf("%s: index %s oversteps it's authority by defining version for %s", reg.Name, idx.Path, key) } } // add resources to registry err = reg.AddResources(cleanedData, false, true, idx.PreRelease) if err != nil { log.Warningf("%s: failed to add resources: %s", reg.Name, err) } } else { log.Debugf("%s: index %s is empty", reg.Name, idx.Path) } // Check if dest dir exists. indexDir := filepath.FromSlash(path.Dir(idx.Path)) err = reg.storageDir.EnsureRelPath(indexDir) if err != nil { log.Warningf("%s: failed to ensure directory for updated index %s: %s", reg.Name, idx.Path, err) } // Index files must be readable by portmaster-staert with user permissions in order to load the index. err = os.WriteFile( //nolint:gosec filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)), indexData, 0o0644, ) if err != nil { log.Warningf("%s: failed to save updated index %s: %s", reg.Name, idx.Path, err) } // Write signature file, if we have one. if len(sigFileData) > 0 { err = os.WriteFile( //nolint:gosec filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)+filesig.Extension), sigFileData, 0o0644, ) if err != nil { log.Warningf("%s: failed to save updated index signature %s: %s", reg.Name, idx.Path+filesig.Extension, err) } } log.Infof("%s: updated index %s with %d entries", reg.Name, idx.Path, len(indexFile.Releases)) return nil } // DownloadUpdates checks if updates are available and downloads updates of used components. func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { // create list of downloads var toUpdate []*ResourceVersion var missingSigs []*ResourceVersion reg.RLock() for _, res := range reg.resources { res.Lock() // check if we want to download if res.inUse() || res.available() || // resource was used in the past utils.StringInSlice(reg.MandatoryUpdates, res.Identifier) { // resource is mandatory // add all non-available and eligible versions to update queue for _, rv := range res.Versions { switch { case !rv.CurrentRelease: // We are not interested in older releases. case !rv.Available: // File is not available. toUpdate = append(toUpdate, rv) case !rv.SigAvailable && res.VerificationOptions != nil: // File signature is not available and verification is enabled. missingSigs = append(missingSigs, rv) } } } res.Unlock() } reg.RUnlock() // nothing to update if len(toUpdate) == 0 && len(missingSigs) == 0 { log.Infof("%s: everything up to date", reg.Name) return nil } // check download dir if err := reg.tmpDir.Ensure(); err != nil { return fmt.Errorf("could not prepare tmp directory for download: %w", err) } // download updates log.Infof("%s: starting to download %d updates in parallel", reg.Name, len(toUpdate)+len(missingSigs)) var wg sync.WaitGroup wg.Add(len(toUpdate) + len(missingSigs)) client := &http.Client{} for idx := range toUpdate { go func(rv *ResourceVersion) { var err error defer wg.Done() defer func() { if x := recover(); x != nil { log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x) } }() for tries := 0; tries < 3; tries++ { err = reg.fetchFile(ctx, client, rv, tries) if err == nil { rv.Available = true return } } if err != nil { log.Warningf("%s: failed to download %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err) } }(toUpdate[idx]) } for idx := range missingSigs { go func(rv *ResourceVersion) { var err error defer wg.Done() defer func() { if x := recover(); x != nil { log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x) } }() for tries := 0; tries < 3; tries++ { err = reg.fetchMissingSig(ctx, client, rv, tries) if err == nil { rv.SigAvailable = true return } } if err != nil { log.Warningf("%s: failed to download missing sig of %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err) } }(missingSigs[idx]) } wg.Wait() log.Infof("%s: finished downloading updates", reg.Name) return nil }