[WIP] Add CI for building deb,rpm installers

Earthfile

@@ -3,6 +3,7 @@ VERSION --arg-scope-and-set --global-cache 0.8
 ARG --global go_version = 1.22
 ARG --global node_version = 18
 ARG --global rust_version = 1.79
+ARG --global tauri_version = "2.0.0-rc.8"
 ARG --global golangci_lint_version = 1.57.1
 
 ARG --global go_builder_image = "golang:${go_version}-alpine"
@@ -56,15 +57,14 @@ build:
 
     # Build Tauri app binaries:
     # ./dist/linux_amd64/portmaster-app
+    # ./dist/windows_amd64/portmaster-app
+    BUILD +tauri-build --target="x86_64-unknown-linux-gnu"
+    BUILD +tauri-build --target="x86_64-pc-windows-gnu"
+
+    # TODO(vladimir): Build bundles
     # ./dist/linux_amd64/Portmaster-0.1.0-1.x86_64.rpm
     # ./dist/linux_amd64/Portmaster_0.1.0_amd64.deb
-    BUILD +tauri-build --target="x86_64-unknown-linux-gnu"
-    # TODO:
-    # BUILD +tauri-build --target="x86_64-pc-windows-gnu"
-
     # Bild Tauri bundle for Windows:
-    # ./dist/windows_amd64/portmaster-app_vX-X-X.zip
-    BUILD +tauri-build-windows-bundle
 
     # Build UI assets:
     # ./dist/all/assets.zip
@@ -82,7 +82,7 @@ angular-ci:
 
 tauri-ci:
     BUILD +tauri-build --target="x86_64-unknown-linux-gnu"
-    BUILD +tauri-build-windows-bundle
+    BUILD +tauri-build --target="x86_64-pc-windows-gnu"
 
 kext-ci:
     BUILD +kext-build
@@ -349,6 +349,7 @@ angular-project:
     # Save portmaster UI as local artifact.
     IF [ "${project}" = "portmaster" ]
         SAVE ARTIFACT --keep-ts "./${project}.zip" AS LOCAL ${outputDir}/all/${project}-ui.zip
+        SAVE ARTIFACT --keep-ts "./${project}.zip" output/${project}.zip
     END
 
 # Build the angular projects (portmaster-UI and tauri-builtin) in dev mode
@@ -420,7 +421,7 @@ rust-base:
     DO rust+INIT --keep_fingerprints=true
 
     # For now we need tauri-cli 2.0.0 for bulding
-    DO rust+CARGO --args="install tauri-cli --version ^2.0.0-beta"
+    DO rust+CARGO --args="install tauri-cli --version ${tauri_version} --locked"
 
     # Explicitly cache here.
     SAVE IMAGE --cache-hint
@@ -434,7 +435,7 @@ tauri-src:
     # are preserved such that Rust's incremental compilation works correctly.
     COPY --keep-ts ./desktop/tauri/ .
     COPY assets/data ./../../assets/data
-    COPY packaging/linux ./../../packaging/linux
+    COPY packaging ./../../packaging
     COPY (+angular-project/output/tauri-builtin --project=tauri-builtin --dist=./dist/tauri-builtin --configuration=production --baseHref="/") ./../angular/dist/tauri-builtin
 
     WORKDIR /app/tauri/src-tauri
@@ -446,31 +447,14 @@ tauri-build:
     FROM +tauri-src
 
     ARG --required target
-    ARG output=".*/release/(([^\./]+|([^\./]+\.(dll|exe)))|bundle/(deb|rpm)/.*\.(deb|rpm))"
-    ARG bundle="none"
-
-    # if we want tauri to create the installer bundles we also need to provide all external binaries
-    # we need to do some magic here because tauri expects the binaries to include the rust target tripple.
-    # We already know that triple because it's a required argument. From that triple, we use +RUST_TO_GO_ARCH_STRING
-    # function from below to parse the triple and guess wich GOOS and GOARCH we need.
-    RUN mkdir /tmp/gobuild
-    RUN mkdir ./binaries
 
+    ARG output=".*/release/([^\./]+|([^\./]+\.(dll|exe)))"
     DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
     RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
 
-    # Just for debugging ...
-    # RUN ls -R ./binaries
-
-    # The following is exected to work but doesn't. for whatever reason cargo-sweep errors out on the windows-toolchain.
-    #
-    #   DO rust+CARGO --args="tauri build --bundles none --ci --target=${target}" --output="release/[^/\.]+"
-    #
-    # For, now, we just directly mount the rust target cache and call cargo ourself.
-
     DO rust+SET_CACHE_MOUNTS_ENV
     RUN rustup target add "${target}"
-    RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build  --ci --target="${target}"
+    RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build  --ci --target="${target}" --no-bundle
     DO rust+COPY_OUTPUT --output="${output}"
 
     # BUG(cross-compilation):
@@ -489,127 +473,13 @@ tauri-build:
     RUN echo output: $(ls -R "target/${target}/release")
 
     # Binaries
-    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster-app"
+    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster"
-    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster-app.exe"
+    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster.exe"
-    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/WebView2Loader.dll" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/WebView2Loader.dll"
+    # SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/WebView2Loader.dll" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/WebView2Loader.dll"
 
-    # Installers
+    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" ./output/portmaster
-    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/deb/*.deb" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
+    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" ./output/portmaster.exe
-    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/rpm/*.rpm" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
 
-tauri-build-windows-bundle:
-    FROM +tauri-src
-
-    ARG target="x86_64-pc-windows-gnu"
-    ARG output=".*/release/(([^\./]+|([^\./]+\.(dll|exe))))"
-    ARG bundle="none"
-
-    ARG GOOS=windows
-    ARG GOARCH=amd64
-    ARG GOARM
-
-    # The binaries will not be used but we still need to create them. Tauri will check for them.
-    RUN mkdir /tmp/gobuild
-    RUN mkdir ./binaries
-
-    DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
-    RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
-
-    # Our tauri app has externalBins configured so tauri will look for them when it finished compiling
-    # the app. Make sure we copy portmaster-start and portmaster-core in all architectures supported.
-    # See documentation for externalBins for more information on how tauri searches for the binaries.
-    COPY (+go-build/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild
-
-    # Place them in the correct folder with the rust target tripple attached.
-    FOR bin IN $(ls /tmp/gobuild)
-        # ${bin$.*} does not work in SET commands unfortunately so we use a shell
-        # snippet here:
-        RUN set -e ; \
-            dest="./binaries/${bin}-${target}" ; \
-            if [ -z "${bin##*.exe}" ]; then \
-                dest="./binaries/${bin%.*}-${target}.exe" ; \
-            fi ; \
-            cp "/tmp/gobuild/${bin}" "${dest}" ;
-    END
-
-    # Just for debugging ...
-    # RUN ls -R ./binaries
-
-    DO rust+SET_CACHE_MOUNTS_ENV
-    RUN rustup target add "${target}"
-    RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --no-bundle --ci --target="${target}"
-    DO rust+COPY_OUTPUT --output="${output}"
-
-    # Get version from git.
-    COPY .git .
-    LET version = "$(git tag --points-at || true)"
-    IF [ -z "${version}" ]
-        LET dev_version = "$(git describe --tags --first-parent --abbrev=0 || true)"
-        IF [ -n "${dev_version}" ]
-            SET version = "${dev_version}"
-        END
-    END
-    IF [ -z "${version}" ]
-        SET version = "v0.0.0"
-    END
-    ENV VERSION="${version}"
-    RUN echo "Version: $VERSION"
-    ENV VERSION_SUFFIX="$(echo $VERSION | tr '.' '-')"
-    RUN echo "Version Suffix: $VERSION_SUFFIX"
-
-    RUN echo output: $(ls -R "target/${target}/release")
-    RUN mv "target/${target}/release/portmaster.exe" "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.exe"
-    RUN zip "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.zip" "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.exe" -j portmaster-app${VERSION_SUFFIX}.exe "target/${target}/release/WebView2Loader.dll" -j WebView2Loader.dll
-    SAVE ARTIFACT --if-exists "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.zip" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
-
-tauri-prep-windows:
-    FROM +angular-base --configuration=production
-    ARG target="x86_64-pc-windows-msvc"
-
-    # if we want tauri to create the installer bundles we also need to provide all external binaries
-    # we need to do some magic here because tauri expects the binaries to include the rust target tripple.
-    # We already know that triple because it's a required argument. From that triple, we use +RUST_TO_GO_ARCH_STRING
-    # function from below to parse the triple and guess wich GOOS and GOARCH we need.
-    RUN mkdir /tmp/gobuild
-    RUN mkdir ./binaries
-
-    DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
-    RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
-
-    # Our tauri app has externalBins configured so tauri will try to embed them when it finished compiling
-    # the app. Make sure we copy portmaster-start and portmaster-core in all architectures supported.
-    # See documentation for externalBins for more information on how tauri searches for the binaries.
-
-    COPY (+go-build/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild
-
-    # Place them in the correct folder with the rust target tripple attached.
-    FOR bin IN $(ls /tmp/gobuild)
-        # ${bin$.*} does not work in SET commands unfortunately so we use a shell
-        # snippet here:
-        RUN set -e ; \
-            dest="./binaries/${bin}-${target}" ; \
-            if [ -z "${bin##*.exe}" ]; then \
-                dest="./binaries/${bin%.*}-${target}.exe" ; \
-            fi ; \
-            cp "/tmp/gobuild/${bin}" "${dest}" ;
-    END
-
-    # Copy source
-    COPY --keep-ts ./desktop/tauri/src-tauri src-tauri
-    COPY --keep-ts ./assets assets
-
-    # Build UI
-    ENV NODE_ENV="production"
-    RUN --no-cache ./node_modules/.bin/ng build --configuration production --base-href / "tauri-builtin"
-
-    # Just for debugging ...
-    # RUN ls -R ./binaries
-    # RUN ls -R ./dist
-
-    SAVE ARTIFACT "./dist/tauri-builtin" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/angular/dist/"
-    SAVE ARTIFACT "./src-tauri" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/tauri/src-tauri"
-    SAVE ARTIFACT "./binaries" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/tauri/src-tauri/"
-    SAVE ARTIFACT "./assets" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/assets"
 
 tauri-release:
     FROM ${work_image}
@@ -624,11 +494,78 @@ tauri-lint:
     ARG target="x86_64-unknown-linux-gnu"
 
     WORKDIR /app
+    # Copy static files that are embedded inside the executable.
     COPY --keep-ts ./assets ./assets
+
+    # Copy all the rust code
     COPY --keep-ts ./desktop/tauri ./desktop/tauri
+
+    # Create a empty ui dir so it will satisfy the build.
     RUN mkdir -p ./desktop/angular/dist/tauri-builtin
+
+    SAVE IMAGE --cache-hint
+
+    # Run the linter.
     WORKDIR /app/desktop/tauri/src-tauri
-    RUN cargo clippy --all-targets --all-features -- -D warnings
+    RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo clippy --all-targets --all-features -- -D warnings
+
+tauri-bundle-linux:
+    FROM +rust-base
+    # ARG --required target
+    ARG target="x86_64-unknown-linux-gnu"
+
+    WORKDIR /app/tauri
+    COPY --keep-ts ./desktop/tauri/ .
+    COPY assets/data ./../../assets/data
+    COPY packaging ./../../packaging
+
+    WORKDIR /app/tauri/src-tauri
+
+    SAVE IMAGE --cache-hint
+
+    
+    DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
+
+    # Build and copy the binaries
+    RUN mkdir -p target/${target}/release
+    COPY (+tauri-build/output/portmaster --target=x86_64-unknown-linux-gnu) ./target/${target}/release/portmaster
+
+
+    RUN mkdir -p binary
+    COPY (+go-build/output/portmaster-core --GOARCH=amd64 --GOOS=linux --CMDS=portmaster-core) ./binary/portmaster-core
+
+    COPY (+assets/assets.zip) ./binary/assets.zip
+    COPY (+angular-project/output/portmaster.zip --project=portmaster --dist=./dist --configuration=production --baseHref=/ui/modules/portmaster/) ./binary/portmaster.zip
+   
+
+    # Download the intel data
+    RUN mkdir -p intel
+
+    RUN wget -O ./intel/geoipv4.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv4_v20240529-0-1.mmdb.gz" && \
+        wget -O ./intel/geoipv6.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv6_v20240529-0-1.mmdb.gz" && \
+        gzip -d ./intel/geoipv4.mmdb.gz && \
+        gzip -d ./intel/geoipv6.mmdb.gz
+
+    RUN touch "./intel/index.dsd"
+    RUN touch "./intel/base.dsdl"
+    RUN touch "./intel/intermediate.dsdl"
+    RUN touch "./intel/urgent.dsdl"
+
+
+    # Generate index files
+    COPY (+go-build/output/updatemgr --GOARCH=amd64 --GOOS=linux --CMDS=updatemgr) ./updatemgr
+    RUN ./updatemgr -dir "./binary" -name "Binary" > ./binary/bin-index.json
+    RUN ./updatemgr -dir "./intel" -name "Intel" > ./intel/intel-index.json
+
+    RUN cat ./binary/bin-index.json
+    RUN cat ./intel/intel-index.json
+
+    # build the installers
+    RUN cargo tauri bundle --ci --target="${target}"
+
+    # Installers
+    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/deb/*.deb" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
+    SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/rpm/*.rpm" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
 
 kext-build:
     FROM ${rust_builder_image}

cmds/updatemgr/confirm.go

@@ -1,20 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"fmt"
-	"os"
-	"strings"
-)
-
-func confirm(msg string) bool {
-	fmt.Printf("%s: [y|n] ", msg)
-
-	scanner := bufio.NewScanner(os.Stdin)
-	ok := scanner.Scan()
-	if ok && strings.TrimSpace(scanner.Text()) == "y" {
-		return true
-	}
-
-	return false
-}

cmds/updatemgr/main.go

@@ -1,58 +1,51 @@
 package main
 
 import (
+	"encoding/json"
+	"flag"
 	"fmt"
 	"os"
-	"path/filepath"
 
-	"github.com/spf13/cobra"
+	"github.com/safing/portmaster/service/updates"
-
-	"github.com/safing/portmaster/base/updater"
-	"github.com/safing/portmaster/base/utils"
 )
 
-var (
+var binaryMap = map[string]updates.Artifact{
-	registry *updater.ResourceRegistry
+	"portmaster-core": {
-	distDir  string
+		Platform: "linux_amd64",
-)
+	},
-
+	"portmaster-core.exe": {
-var rootCmd = &cobra.Command{
+		Platform: "windows_amd64",
-	Use:   "updatemgr",
+	},
-	Short: "A simple tool to assist in the update and release process",
+	"portmaster-kext.sys": {
-	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		Platform: "windows_amd64",
-		// Check if the distribution directory exists.
-		absDistPath, err := filepath.Abs(distDir)
-		if err != nil {
-			return fmt.Errorf("failed to get absolute path of distribution directory: %w", err)
-		}
-		_, err = os.Stat(absDistPath)
-		if err != nil {
-			return fmt.Errorf("failed to access distribution directory: %w", err)
-		}
-
-		registry = &updater.ResourceRegistry{}
-		err = registry.Initialize(utils.NewDirStructure(absDistPath, 0o0755))
-		if err != nil {
-			return err
-		}
-
-		err = registry.ScanStorage("")
-		if err != nil {
-			return err
-		}
-
-		return nil
 	},
-	SilenceUsage: true,
-}
-
-func init() {
-	flags := rootCmd.PersistentFlags()
-	flags.StringVar(&distDir, "dist-dir", "dist", "Set the distribution directory. Falls back to ./dist if available.")
 }
 
 func main() {
-	if err := rootCmd.Execute(); err != nil {
+	dir := flag.String("dir", "", "path to the directory that contains the artifacts")
-		os.Exit(1)
+	name := flag.String("name", "", "name of the bundle")
+	version := flag.String("version", "", "version of the bundle")
+
+	flag.Parse()
+	if *dir == "" {
+		fmt.Fprintf(os.Stderr, "-dir parameter is required\n")
+		return
 	}
+	if *name == "" {
+		fmt.Fprintf(os.Stderr, "-name parameter is required\n")
+		return
+	}
+
+	bundle, err := updates.GenerateBundleFromDir(*name, *version, binaryMap, *dir)
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to generate bundle: %s\n", err)
+		return
+	}
+
+	bundleStr, err := json.MarshalIndent(&bundle, "", "  ")
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "failed to marshal bundle: %s\n", err)
+	}
+
+	fmt.Printf("%s", bundleStr)
 }

cmds/updatemgr/purge.go

@@ -1,33 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/spf13/cobra"
-
-	"github.com/safing/portmaster/base/log"
-)
-
-func init() {
-	rootCmd.AddCommand(purgeCmd)
-}
-
-var purgeCmd = &cobra.Command{
-	Use:   "purge",
-	Short: "Remove old resource versions that are superseded by at least three versions",
-	RunE:  purge,
-}
-
-func purge(cmd *cobra.Command, args []string) error {
-	log.SetLogLevel(log.TraceLevel)
-	err := log.Start()
-	if err != nil {
-		fmt.Printf("failed to start logging: %s\n", err)
-	}
-	defer log.Shutdown()
-
-	registry.SelectVersions()
-	registry.Purge(3)
-
-	return nil
-}

cmds/updatemgr/release.go

@@ -1,195 +0,0 @@
-package main
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"os"
-	"path/filepath"
-	"time"
-
-	"github.com/spf13/cobra"
-
-	"github.com/safing/portmaster/base/updater"
-)
-
-var (
-	releaseCmd = &cobra.Command{
-		Use:   "release",
-		Short: "Release scans the distribution directory and creates registry indexes and the symlink structure",
-		Args:  cobra.ExactArgs(1),
-		RunE:  release,
-	}
-	preReleaseCmd = &cobra.Command{
-		Use:   "prerelease",
-		Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes",
-		Args:  cobra.ExactArgs(1),
-		RunE:  release,
-	}
-	preReleaseFrom   string
-	resetPreReleases bool
-)
-
-func init() {
-	rootCmd.AddCommand(releaseCmd)
-	rootCmd.AddCommand(preReleaseCmd)
-
-	preReleaseCmd.Flags().StringVar(&preReleaseFrom, "from", "", "Make a pre-release based on the given channel")
-	_ = preReleaseCmd.MarkFlagRequired("from")
-	preReleaseCmd.Flags().BoolVar(&resetPreReleases, "reset", false, "Reset pre-release assets")
-}
-
-func release(cmd *cobra.Command, args []string) error {
-	channel := args[0]
-
-	// Check if we want to reset instead.
-	if resetPreReleases {
-		return removeFilesFromIndex(getChannelVersions(preReleaseFrom, true))
-	}
-
-	// Write new index.
-	err := writeIndex(
-		channel,
-		getChannelVersions(preReleaseFrom, false),
-	)
-	if err != nil {
-		return err
-	}
-
-	// Only when doing a release:
-	if preReleaseFrom == "" {
-		// Create symlinks to latest stable versions.
-		if !confirm("\nDo you want to write latest symlinks?") {
-			fmt.Println("aborted...")
-			return nil
-		}
-		symlinksDir := registry.StorageDir().ChildDir("latest", 0o755)
-		err = registry.CreateSymlinks(symlinksDir)
-		if err != nil {
-			return err
-		}
-		fmt.Println("written latest symlinks")
-	}
-
-	return nil
-}
-
-func writeIndex(channel string, versions map[string]string) error {
-	// Create new index file.
-	indexFile := &updater.IndexFile{
-		Channel:   channel,
-		Published: time.Now().UTC().Round(time.Second),
-		Releases:  versions,
-	}
-
-	// Export versions and format them.
-	confirmData, err := json.MarshalIndent(indexFile, "", " ")
-	if err != nil {
-		return err
-	}
-
-	// Build index paths.
-	oldIndexPath := filepath.Join(registry.StorageDir().Path, channel+".json")
-	newIndexPath := filepath.Join(registry.StorageDir().Path, channel+".v2.json")
-
-	// Print preview.
-	fmt.Printf("%s\n%s\n%s\n\n", channel, oldIndexPath, newIndexPath)
-	fmt.Println(string(confirmData))
-
-	// Ask for confirmation.
-	if !confirm("\nDo you want to write this index?") {
-		fmt.Println("aborted...")
-		return nil
-	}
-
-	// Write indexes.
-	err = writeAsJSON(oldIndexPath, versions)
-	if err != nil {
-		return fmt.Errorf("failed to write %s: %w", oldIndexPath, err)
-	}
-	err = writeAsJSON(newIndexPath, indexFile)
-	if err != nil {
-		return fmt.Errorf("failed to write %s: %w", newIndexPath, err)
-	}
-
-	return nil
-}
-
-func writeAsJSON(path string, data any) error {
-	// Marshal to JSON.
-	jsonData, err := json.MarshalIndent(data, "", " ")
-	if err != nil {
-		return err
-	}
-
-	// Write to disk.
-	err = os.WriteFile(path, jsonData, 0o0644) //nolint:gosec
-	if err != nil {
-		return err
-	}
-
-	fmt.Printf("written %s\n", path)
-	return nil
-}
-
-func removeFilesFromIndex(versions map[string]string) error {
-	// Print preview.
-	fmt.Println("To be deleted:")
-	for _, filePath := range versions {
-		fmt.Println(filePath)
-	}
-
-	// Ask for confirmation.
-	if !confirm("\nDo you want to delete these files?") {
-		fmt.Println("aborted...")
-		return nil
-	}
-
-	// Delete files.
-	for _, filePath := range versions {
-		err := os.Remove(filePath)
-		if err != nil {
-			return err
-		}
-	}
-	fmt.Println("deleted")
-
-	return nil
-}
-
-func getChannelVersions(prereleaseFrom string, storagePath bool) map[string]string {
-	if prereleaseFrom != "" {
-		registry.AddIndex(updater.Index{
-			Path:       prereleaseFrom + ".json",
-			PreRelease: false,
-		})
-		err := registry.LoadIndexes(context.Background())
-		if err != nil {
-			panic(err)
-		}
-	}
-
-	// Sort all versions.
-	registry.SelectVersions()
-	export := registry.Export()
-
-	// Go through all versions and save the highest version, if not stable or beta.
-	versions := make(map[string]string)
-	for _, rv := range export {
-		highestVersion := rv.Versions[0]
-
-		// Ignore versions that are in the reference release channel.
-		if highestVersion.CurrentRelease {
-			continue
-		}
-
-		// Add highest version of matching release channel.
-		if storagePath {
-			versions[rv.Identifier] = rv.GetFile().Path()
-		} else {
-			versions[rv.Identifier] = highestVersion.VersionNumber
-		}
-	}
-
-	return versions
-}

cmds/updatemgr/scan.go

@@ -1,49 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-
-	"github.com/spf13/cobra"
-)
-
-func init() {
-	rootCmd.AddCommand(scanCmd)
-}
-
-var scanCmd = &cobra.Command{
-	Use:   "scan",
-	Short: "Scan the specified directory and print the result",
-	RunE:  scan,
-}
-
-func scan(cmd *cobra.Command, args []string) error {
-	// Reset and rescan.
-	registry.ResetResources()
-	err := registry.ScanStorage("")
-	if err != nil {
-		return err
-	}
-
-	// Export latest versions.
-	data, err := json.MarshalIndent(exportSelected(true), "", " ")
-	if err != nil {
-		return err
-	}
-	// Print them.
-	fmt.Println(string(data))
-
-	return nil
-}
-
-func exportSelected(preReleases bool) map[string]string {
-	registry.SetUsePreReleases(preReleases)
-	registry.SelectVersions()
-	export := registry.Export()
-
-	versions := make(map[string]string)
-	for _, rv := range export {
-		versions[rv.Identifier] = rv.SelectedVersion.VersionNumber
-	}
-	return versions
-}

cmds/updatemgr/sign.go

@@ -1,303 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"io/fs"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/spf13/cobra"
-
-	"github.com/safing/jess"
-	"github.com/safing/jess/filesig"
-	"github.com/safing/jess/truststores"
-)
-
-func init() {
-	rootCmd.AddCommand(signCmd)
-
-	// Required argument: envelope
-	signCmd.PersistentFlags().StringVarP(&envelopeName, "envelope", "", "",
-		"specify envelope name used for signing",
-	)
-	_ = signCmd.MarkFlagRequired("envelope")
-
-	// Optional arguments: verbose, tsdir, tskeyring
-	signCmd.PersistentFlags().BoolVarP(&signVerbose, "verbose", "v", false,
-		"enable verbose output",
-	)
-	signCmd.PersistentFlags().StringVarP(&trustStoreDir, "tsdir", "", "",
-		"specify a truststore directory (default loaded from JESS_TS_DIR env variable)",
-	)
-	signCmd.PersistentFlags().StringVarP(&trustStoreKeyring, "tskeyring", "", "",
-		"specify a truststore keyring namespace (default loaded from JESS_TS_KEYRING env variable) - lower priority than tsdir",
-	)
-
-	// Subcommand for signing indexes.
-	signCmd.AddCommand(signIndexCmd)
-}
-
-var (
-	signCmd = &cobra.Command{
-		Use:   "sign",
-		Short: "Sign resources",
-		RunE:  sign,
-		Args:  cobra.NoArgs,
-	}
-	signIndexCmd = &cobra.Command{
-		Use:   "index",
-		Short: "Sign indexes",
-		RunE:  signIndex,
-		Args:  cobra.ExactArgs(1),
-	}
-
-	envelopeName string
-	signVerbose  bool
-)
-
-func sign(cmd *cobra.Command, args []string) error {
-	// Setup trust store.
-	trustStore, err := setupTrustStore()
-	if err != nil {
-		return err
-	}
-
-	// Get envelope.
-	signingEnvelope, err := trustStore.GetEnvelope(envelopeName)
-	if err != nil {
-		return err
-	}
-
-	// Get all resources and iterate over all versions.
-	export := registry.Export()
-	var verified, signed, fails int
-	for _, rv := range export {
-		for _, version := range rv.Versions {
-			file := version.GetFile()
-
-			// Check if there is an existing signature.
-			_, err := os.Stat(file.Path() + filesig.Extension)
-			switch {
-			case err == nil || errors.Is(err, fs.ErrExist):
-				// If the file exists, just verify.
-				fileData, err := filesig.VerifyFile(
-					file.Path(),
-					file.Path()+filesig.Extension,
-					file.SigningMetadata(),
-					trustStore,
-				)
-				if err != nil {
-					fmt.Printf("[FAIL] signature error for %s: %s\n", file.Path(), err)
-					fails++
-				} else {
-					if signVerbose {
-						fmt.Printf("[ OK ] valid signature for %s: signed by %s\n", file.Path(), getSignedByMany(fileData, trustStore))
-					}
-					verified++
-				}
-
-			case errors.Is(err, fs.ErrNotExist):
-				// Attempt to sign file.
-				fileData, err := filesig.SignFile(
-					file.Path(),
-					file.Path()+filesig.Extension,
-					file.SigningMetadata(),
-					signingEnvelope,
-					trustStore,
-				)
-				if err != nil {
-					fmt.Printf("[FAIL] failed to sign %s: %s\n", file.Path(), err)
-					fails++
-				} else {
-					fmt.Printf("[SIGN] signed %s with %s\n", file.Path(), getSignedBySingle(fileData, trustStore))
-					signed++
-				}
-
-			default:
-				// File access error.
-				fmt.Printf("[FAIL] failed to access %s: %s\n", file.Path(), err)
-				fails++
-			}
-		}
-	}
-
-	if verified > 0 {
-		fmt.Printf("[STAT] verified %d files\n", verified)
-	}
-	if signed > 0 {
-		fmt.Printf("[STAT] signed %d files\n", signed)
-	}
-	if fails > 0 {
-		return fmt.Errorf("signing or verification failed on %d files", fails)
-	}
-	return nil
-}
-
-func signIndex(cmd *cobra.Command, args []string) error {
-	// Setup trust store.
-	trustStore, err := setupTrustStore()
-	if err != nil {
-		return err
-	}
-
-	// Get envelope.
-	signingEnvelope, err := trustStore.GetEnvelope(envelopeName)
-	if err != nil {
-		return err
-	}
-
-	// Resolve globs.
-	files := make([]string, 0, len(args))
-	for _, arg := range args {
-		matches, err := filepath.Glob(arg)
-		if err != nil {
-			return err
-		}
-		files = append(files, matches...)
-	}
-
-	// Go through all files.
-	var verified, signed, fails int
-	for _, file := range files {
-		sigFile := file + filesig.Extension
-
-		// Ignore matches for the signatures.
-		if strings.HasSuffix(file, filesig.Extension) {
-			continue
-		}
-
-		// Check if there is an existing signature.
-		_, err := os.Stat(sigFile)
-		switch {
-		case err == nil || errors.Is(err, fs.ErrExist):
-			// If the file exists, just verify.
-			fileData, err := filesig.VerifyFile(
-				file,
-				sigFile,
-				nil,
-				trustStore,
-			)
-			if err == nil {
-				if signVerbose {
-					fmt.Printf("[ OK ] valid signature for %s: signed by %s\n", file, getSignedByMany(fileData, trustStore))
-				}
-				verified++
-
-				// Indexes are expected to change, so just sign the index again if verification fails.
-				continue
-			}
-
-			fallthrough
-		case errors.Is(err, fs.ErrNotExist):
-			// Attempt to sign file.
-			fileData, err := filesig.SignFile(
-				file,
-				sigFile,
-				nil,
-				signingEnvelope,
-				trustStore,
-			)
-			if err != nil {
-				fmt.Printf("[FAIL] failed to sign %s: %s\n", file, err)
-				fails++
-			} else {
-				fmt.Printf("[SIGN] signed %s with %s\n", file, getSignedBySingle(fileData, trustStore))
-				signed++
-			}
-
-		default:
-			// File access error.
-			fmt.Printf("[FAIL] failed to access %s: %s\n", sigFile, err)
-			fails++
-		}
-	}
-
-	if verified > 0 {
-		fmt.Printf("[STAT] verified %d files", verified)
-	}
-	if signed > 0 {
-		fmt.Printf("[STAT] signed %d files", signed)
-	}
-	if fails > 0 {
-		return fmt.Errorf("signing failed on %d files", fails)
-	}
-	return nil
-}
-
-var (
-	trustStoreDir     string
-	trustStoreKeyring string
-)
-
-func setupTrustStore() (trustStore truststores.ExtendedTrustStore, err error) {
-	// Get trust store directory.
-	if trustStoreDir == "" {
-		trustStoreDir, _ = os.LookupEnv("JESS_TS_DIR")
-		if trustStoreDir == "" {
-			trustStoreDir, _ = os.LookupEnv("JESS_TSDIR")
-		}
-	}
-	if trustStoreDir != "" {
-		trustStore, err = truststores.NewDirTrustStore(trustStoreDir)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	// Get trust store keyring.
-	if trustStore == nil {
-		if trustStoreKeyring == "" {
-			trustStoreKeyring, _ = os.LookupEnv("JESS_TS_KEYRING")
-			if trustStoreKeyring == "" {
-				trustStoreKeyring, _ = os.LookupEnv("JESS_TSKEYRING")
-			}
-		}
-		if trustStoreKeyring != "" {
-			trustStore, err = truststores.NewKeyringTrustStore(trustStoreKeyring)
-			if err != nil {
-				return nil, err
-			}
-		}
-	}
-
-	// Truststore is mandatory.
-	if trustStore == nil {
-		return nil, errors.New("no truststore configured, please pass arguments or use env variables")
-	}
-
-	return trustStore, nil
-}
-
-func getSignedByMany(fds []*filesig.FileData, trustStore jess.TrustStore) string {
-	signedBy := make([]string, 0, len(fds))
-	for _, fd := range fds {
-		if sig := fd.Signature(); sig != nil {
-			for _, seal := range sig.Signatures {
-				if signet, err := trustStore.GetSignet(seal.ID, true); err == nil {
-					signedBy = append(signedBy, fmt.Sprintf("%s (%s)", signet.Info.Name, seal.ID))
-				} else {
-					signedBy = append(signedBy, seal.ID)
-				}
-			}
-		}
-	}
-	return strings.Join(signedBy, " and ")
-}
-
-func getSignedBySingle(fd *filesig.FileData, trustStore jess.TrustStore) string {
-	if sig := fd.Signature(); sig != nil {
-		signedBy := make([]string, 0, len(sig.Signatures))
-		for _, seal := range sig.Signatures {
-			if signet, err := trustStore.GetSignet(seal.ID, true); err == nil {
-				signedBy = append(signedBy, fmt.Sprintf("%s (%s)", signet.Info.Name, seal.ID))
-			} else {
-				signedBy = append(signedBy, seal.ID)
-			}
-		}
-		return strings.Join(signedBy, " and ")
-	}
-
-	return ""
-}

desktop/tauri/src-tauri/tauri.conf.json5

@@ -59,18 +59,25 @@
         ],
         "desktopTemplate": "../../../packaging/linux/portmaster.desktop",
         "files": {
+          // Service file
           "/usr/lib/systemd/system/portmaster.service": "../../../packaging/linux/portmaster.service",
-          "/usr/lib/portmaster/bin-index.json": "binaries/bin-index.json",
+
-          "/usr/lib/portmaster/portmaster-core": "binaries/portmaster-core",
+          // Binary files
-          "/usr/lib/portmaster/portmaster.zip": "binaries/portmaster.zip",
+          "/usr/lib/portmaster/bin-index.json": "binary/bin-index.json",
-          "/usr/lib/portmaster/assets.zip": "binaries/assets.zip",
+          "/usr/lib/portmaster/portmaster-core": "binary/portmaster-core",
-          "/var/lib/portmaster/intel/intel-index.json": "binaries/intel-index.json",
+          "/usr/lib/portmaster/portmaster.zip": "binary/portmaster.zip",
-          "/var/lib/portmaster/intel/base.dsdl": "binaries/base.dsdl",
+          "/usr/lib/portmaster/assets.zip": "binary/assets.zip",
-          "/var/lib/portmaster/intel/geoipv4.mmdb": "binaries/geoipv4.mmdb",
+
-          "/var/lib/portmaster/intel/geoipv6.mmdb": "binaries/geoipv6.mmdb",
+          // Intel files
-          "/var/lib/portmaster/intel/index.dsd": "binaries/index.dsd",
+          "/var/lib/portmaster/intel/intel-index.json": "intel/intel-index.json",
-          "/var/lib/portmaster/intel/intermediate.dsdl": "binaries/intermediate.dsdl",
+          "/var/lib/portmaster/intel/base.dsdl": "intel/base.dsdl",
-          "/var/lib/portmaster/intel/urgent.dsdl": "binaries/urgent.dsdl",
+          "/var/lib/portmaster/intel/geoipv4.mmdb": "intel/geoipv4.mmdb",
+          "/var/lib/portmaster/intel/geoipv6.mmdb": "intel/geoipv6.mmdb",
+          "/var/lib/portmaster/intel/index.dsd": "intel/index.dsd",
+          "/var/lib/portmaster/intel/intermediate.dsdl": "intel/intermediate.dsdl",
+          "/var/lib/portmaster/intel/urgent.dsdl": "intel/urgent.dsdl",
+
+          // Shortcut
           "/etc/xdg/autostart/portmaster.desktop": "../../../packaging/linux/portmaster-autostart.desktop"
         },
         "postInstallScript": "../../../packaging/linux/postinst",
@@ -83,18 +90,25 @@
         "desktopTemplate": "../../../packaging/linux/portmaster.desktop",
         "release": "1",
         "files": {
+          // Service file
           "/usr/lib/systemd/system/portmaster.service": "../../../packaging/linux/portmaster.service",
-          "/usr/lib/portmaster/bin-index.json": "binaries/bin-index.json",
+
-          "/usr/lib/portmaster/portmaster-core": "binaries/portmaster-core",
+          // Binary files
-          "/usr/lib/portmaster/portmaster.zip": "binaries/portmaster.zip",
+          "/usr/lib/portmaster/bin-index.json": "binary/bin-index.json",
-          "/usr/lib/portmaster/assets.zip": "binaries/assets.zip",
+          "/usr/lib/portmaster/portmaster-core": "binary/portmaster-core",
-          "/var/lib/portmaster/intel/intel-index.json": "binaries/intel-index.json",
+          "/usr/lib/portmaster/portmaster.zip": "binary/portmaster.zip",
-          "/var/lib/portmaster/intel/base.dsdl": "binaries/base.dsdl",
+          "/usr/lib/portmaster/assets.zip": "binary/assets.zip",
-          "/var/lib/portmaster/intel/geoipv4.mmdb": "binaries/geoipv4.mmdb",
+
-          "/var/lib/portmaster/intel/geoipv6.mmdb": "binaries/geoipv6.mmdb",
+          // Intel files
-          "/var/lib/portmaster/intel/index.dsd": "binaries/index.dsd",
+          "/var/lib/portmaster/intel/intel-index.json": "intel/intel-index.json",
-          "/var/lib/portmaster/intel/intermediate.dsdl": "binaries/intermediate.dsdl",
+          "/var/lib/portmaster/intel/base.dsdl": "intel/base.dsdl",
-          "/var/lib/portmaster/intel/urgent.dsdl": "binaries/urgent.dsdl",
+          "/var/lib/portmaster/intel/geoipv4.mmdb": "intel/geoipv4.mmdb",
+          "/var/lib/portmaster/intel/geoipv6.mmdb": "intel/geoipv6.mmdb",
+          "/var/lib/portmaster/intel/index.dsd": "intel/index.dsd",
+          "/var/lib/portmaster/intel/intermediate.dsdl": "intel/intermediate.dsdl",
+          "/var/lib/portmaster/intel/urgent.dsdl": "intel/urgent.dsdl",
+
+          // Shortcut
           "/etc/xdg/autostart/portmaster.desktop": "../../../packaging/linux/portmaster-autostart.desktop"
         },
         "postInstallScript": "../../../packaging/linux/postinst",

service/updates/bundle.go

@@ -120,3 +120,57 @@ func checkIfFileIsValid(filename string, artifact Artifact) (bool, error) {
 	}
 	return true, nil
 }
+
+// GenerateBundleFromDir generates a bundle from a given folder.
+func GenerateBundleFromDir(name string, version string, properties map[string]Artifact, dirPath string) (*Bundle, error) {
+	files, err := os.ReadDir(dirPath)
+	if err != nil {
+		return nil, err
+	}
+	artifacts := make([]Artifact, 0, len(files))
+	for _, f := range files {
+		// Skip dirs
+		if f.IsDir() {
+			continue
+		}
+
+		artifact := Artifact{
+			Filename: f.Name(),
+		}
+		predefined, ok := properties[f.Name()]
+		// Check if caller supplied predefined settings for this artifact.
+		if ok {
+			// File that have compression may have different filename and artifact filename. (because of the extension)
+			// If caller did not specify the artifact filename set it as the same as the filename.
+			if predefined.Filename == "" {
+				predefined.Filename = f.Name()
+			}
+			artifact = predefined
+		}
+		content, err := os.ReadFile(filepath.Join(dirPath, f.Name()))
+		if err != nil {
+			return nil, err
+		}
+
+		// Decompress if compression was applied to the file.
+		if artifact.Unpack != "" {
+			content, err = unpack(artifact.Unpack, content)
+			if err != nil {
+				return nil, err
+			}
+		}
+
+		hash := sha256.Sum256(content)
+		hashStr := hex.EncodeToString(hash[:])
+		artifact.SHA256 = hashStr
+
+		artifacts = append(artifacts, artifact)
+	}
+
+	return &Bundle{
+		Name:      name,
+		Version:   version,
+		Artifacts: artifacts,
+		Published: time.Now(),
+	}, nil
+}