From 8e1f3c0ed95f231e15cf0363292b1a35bd9129a7 Mon Sep 17 00:00:00 2001 From: Vladimir Stoilov Date: Wed, 2 Oct 2024 15:37:39 +0300 Subject: [PATCH] [WIP] Add CI for building deb,rpm installers --- Earthfile | 233 +++++++---------- cmds/updatemgr/confirm.go | 20 -- cmds/updatemgr/main.go | 81 +++--- cmds/updatemgr/purge.go | 33 --- cmds/updatemgr/release.go | 195 --------------- cmds/updatemgr/scan.go | 49 ---- cmds/updatemgr/sign.go | 303 ----------------------- desktop/tauri/src-tauri/tauri.conf.json5 | 58 +++-- service/updates/bundle.go | 54 ++++ 9 files changed, 212 insertions(+), 814 deletions(-) delete mode 100644 cmds/updatemgr/confirm.go delete mode 100644 cmds/updatemgr/purge.go delete mode 100644 cmds/updatemgr/release.go delete mode 100644 cmds/updatemgr/scan.go delete mode 100644 cmds/updatemgr/sign.go diff --git a/Earthfile b/Earthfile index 3468f6ac..72504d8e 100644 --- a/Earthfile +++ b/Earthfile @@ -3,6 +3,7 @@ VERSION --arg-scope-and-set --global-cache 0.8 ARG --global go_version = 1.22 ARG --global node_version = 18 ARG --global rust_version = 1.79 +ARG --global tauri_version = "2.0.0-rc.8" ARG --global golangci_lint_version = 1.57.1 ARG --global go_builder_image = "golang:${go_version}-alpine" @@ -56,15 +57,14 @@ build: # Build Tauri app binaries: # ./dist/linux_amd64/portmaster-app + # ./dist/windows_amd64/portmaster-app + BUILD +tauri-build --target="x86_64-unknown-linux-gnu" + BUILD +tauri-build --target="x86_64-pc-windows-gnu" + + # TODO(vladimir): Build bundles # ./dist/linux_amd64/Portmaster-0.1.0-1.x86_64.rpm # ./dist/linux_amd64/Portmaster_0.1.0_amd64.deb - BUILD +tauri-build --target="x86_64-unknown-linux-gnu" - # TODO: - # BUILD +tauri-build --target="x86_64-pc-windows-gnu" - # Bild Tauri bundle for Windows: - # ./dist/windows_amd64/portmaster-app_vX-X-X.zip - BUILD +tauri-build-windows-bundle # Build UI assets: # ./dist/all/assets.zip @@ -82,7 +82,7 @@ angular-ci: tauri-ci: BUILD +tauri-build --target="x86_64-unknown-linux-gnu" - BUILD +tauri-build-windows-bundle + BUILD +tauri-build --target="x86_64-pc-windows-gnu" kext-ci: BUILD +kext-build @@ -349,6 +349,7 @@ angular-project: # Save portmaster UI as local artifact. IF [ "${project}" = "portmaster" ] SAVE ARTIFACT --keep-ts "./${project}.zip" AS LOCAL ${outputDir}/all/${project}-ui.zip + SAVE ARTIFACT --keep-ts "./${project}.zip" output/${project}.zip END # Build the angular projects (portmaster-UI and tauri-builtin) in dev mode @@ -420,7 +421,7 @@ rust-base: DO rust+INIT --keep_fingerprints=true # For now we need tauri-cli 2.0.0 for bulding - DO rust+CARGO --args="install tauri-cli --version ^2.0.0-beta" + DO rust+CARGO --args="install tauri-cli --version ${tauri_version} --locked" # Explicitly cache here. SAVE IMAGE --cache-hint @@ -434,7 +435,7 @@ tauri-src: # are preserved such that Rust's incremental compilation works correctly. COPY --keep-ts ./desktop/tauri/ . COPY assets/data ./../../assets/data - COPY packaging/linux ./../../packaging/linux + COPY packaging ./../../packaging COPY (+angular-project/output/tauri-builtin --project=tauri-builtin --dist=./dist/tauri-builtin --configuration=production --baseHref="/") ./../angular/dist/tauri-builtin WORKDIR /app/tauri/src-tauri @@ -446,31 +447,14 @@ tauri-build: FROM +tauri-src ARG --required target - ARG output=".*/release/(([^\./]+|([^\./]+\.(dll|exe)))|bundle/(deb|rpm)/.*\.(deb|rpm))" - ARG bundle="none" - - # if we want tauri to create the installer bundles we also need to provide all external binaries - # we need to do some magic here because tauri expects the binaries to include the rust target tripple. - # We already know that triple because it's a required argument. From that triple, we use +RUST_TO_GO_ARCH_STRING - # function from below to parse the triple and guess wich GOOS and GOARCH we need. - RUN mkdir /tmp/gobuild - RUN mkdir ./binaries + ARG output=".*/release/([^\./]+|([^\./]+\.(dll|exe)))" DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}" RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}" - # Just for debugging ... - # RUN ls -R ./binaries - - # The following is exected to work but doesn't. for whatever reason cargo-sweep errors out on the windows-toolchain. - # - # DO rust+CARGO --args="tauri build --bundles none --ci --target=${target}" --output="release/[^/\.]+" - # - # For, now, we just directly mount the rust target cache and call cargo ourself. - DO rust+SET_CACHE_MOUNTS_ENV RUN rustup target add "${target}" - RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --ci --target="${target}" + RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --ci --target="${target}" --no-bundle DO rust+COPY_OUTPUT --output="${output}" # BUG(cross-compilation): @@ -489,127 +473,13 @@ tauri-build: RUN echo output: $(ls -R "target/${target}/release") # Binaries - SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster-app" - SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster-app.exe" - SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/WebView2Loader.dll" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/WebView2Loader.dll" + SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster" + SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster.exe" + # SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/WebView2Loader.dll" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/WebView2Loader.dll" - # Installers - SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/deb/*.deb" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/" - SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/rpm/*.rpm" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/" + SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" ./output/portmaster + SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" ./output/portmaster.exe -tauri-build-windows-bundle: - FROM +tauri-src - - ARG target="x86_64-pc-windows-gnu" - ARG output=".*/release/(([^\./]+|([^\./]+\.(dll|exe))))" - ARG bundle="none" - - ARG GOOS=windows - ARG GOARCH=amd64 - ARG GOARM - - # The binaries will not be used but we still need to create them. Tauri will check for them. - RUN mkdir /tmp/gobuild - RUN mkdir ./binaries - - DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}" - RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}" - - # Our tauri app has externalBins configured so tauri will look for them when it finished compiling - # the app. Make sure we copy portmaster-start and portmaster-core in all architectures supported. - # See documentation for externalBins for more information on how tauri searches for the binaries. - COPY (+go-build/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild - - # Place them in the correct folder with the rust target tripple attached. - FOR bin IN $(ls /tmp/gobuild) - # ${bin$.*} does not work in SET commands unfortunately so we use a shell - # snippet here: - RUN set -e ; \ - dest="./binaries/${bin}-${target}" ; \ - if [ -z "${bin##*.exe}" ]; then \ - dest="./binaries/${bin%.*}-${target}.exe" ; \ - fi ; \ - cp "/tmp/gobuild/${bin}" "${dest}" ; - END - - # Just for debugging ... - # RUN ls -R ./binaries - - DO rust+SET_CACHE_MOUNTS_ENV - RUN rustup target add "${target}" - RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --no-bundle --ci --target="${target}" - DO rust+COPY_OUTPUT --output="${output}" - - # Get version from git. - COPY .git . - LET version = "$(git tag --points-at || true)" - IF [ -z "${version}" ] - LET dev_version = "$(git describe --tags --first-parent --abbrev=0 || true)" - IF [ -n "${dev_version}" ] - SET version = "${dev_version}" - END - END - IF [ -z "${version}" ] - SET version = "v0.0.0" - END - ENV VERSION="${version}" - RUN echo "Version: $VERSION" - ENV VERSION_SUFFIX="$(echo $VERSION | tr '.' '-')" - RUN echo "Version Suffix: $VERSION_SUFFIX" - - RUN echo output: $(ls -R "target/${target}/release") - RUN mv "target/${target}/release/portmaster.exe" "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.exe" - RUN zip "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.zip" "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.exe" -j portmaster-app${VERSION_SUFFIX}.exe "target/${target}/release/WebView2Loader.dll" -j WebView2Loader.dll - SAVE ARTIFACT --if-exists "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.zip" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/" - -tauri-prep-windows: - FROM +angular-base --configuration=production - ARG target="x86_64-pc-windows-msvc" - - # if we want tauri to create the installer bundles we also need to provide all external binaries - # we need to do some magic here because tauri expects the binaries to include the rust target tripple. - # We already know that triple because it's a required argument. From that triple, we use +RUST_TO_GO_ARCH_STRING - # function from below to parse the triple and guess wich GOOS and GOARCH we need. - RUN mkdir /tmp/gobuild - RUN mkdir ./binaries - - DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}" - RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}" - - # Our tauri app has externalBins configured so tauri will try to embed them when it finished compiling - # the app. Make sure we copy portmaster-start and portmaster-core in all architectures supported. - # See documentation for externalBins for more information on how tauri searches for the binaries. - - COPY (+go-build/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild - - # Place them in the correct folder with the rust target tripple attached. - FOR bin IN $(ls /tmp/gobuild) - # ${bin$.*} does not work in SET commands unfortunately so we use a shell - # snippet here: - RUN set -e ; \ - dest="./binaries/${bin}-${target}" ; \ - if [ -z "${bin##*.exe}" ]; then \ - dest="./binaries/${bin%.*}-${target}.exe" ; \ - fi ; \ - cp "/tmp/gobuild/${bin}" "${dest}" ; - END - - # Copy source - COPY --keep-ts ./desktop/tauri/src-tauri src-tauri - COPY --keep-ts ./assets assets - - # Build UI - ENV NODE_ENV="production" - RUN --no-cache ./node_modules/.bin/ng build --configuration production --base-href / "tauri-builtin" - - # Just for debugging ... - # RUN ls -R ./binaries - # RUN ls -R ./dist - - SAVE ARTIFACT "./dist/tauri-builtin" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/angular/dist/" - SAVE ARTIFACT "./src-tauri" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/tauri/src-tauri" - SAVE ARTIFACT "./binaries" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/tauri/src-tauri/" - SAVE ARTIFACT "./assets" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/assets" tauri-release: FROM ${work_image} @@ -624,11 +494,78 @@ tauri-lint: ARG target="x86_64-unknown-linux-gnu" WORKDIR /app + # Copy static files that are embedded inside the executable. COPY --keep-ts ./assets ./assets + + # Copy all the rust code COPY --keep-ts ./desktop/tauri ./desktop/tauri + + # Create a empty ui dir so it will satisfy the build. RUN mkdir -p ./desktop/angular/dist/tauri-builtin + + SAVE IMAGE --cache-hint + + # Run the linter. WORKDIR /app/desktop/tauri/src-tauri - RUN cargo clippy --all-targets --all-features -- -D warnings + RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo clippy --all-targets --all-features -- -D warnings + +tauri-bundle-linux: + FROM +rust-base + # ARG --required target + ARG target="x86_64-unknown-linux-gnu" + + WORKDIR /app/tauri + COPY --keep-ts ./desktop/tauri/ . + COPY assets/data ./../../assets/data + COPY packaging ./../../packaging + + WORKDIR /app/tauri/src-tauri + + SAVE IMAGE --cache-hint + + + DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}" + + # Build and copy the binaries + RUN mkdir -p target/${target}/release + COPY (+tauri-build/output/portmaster --target=x86_64-unknown-linux-gnu) ./target/${target}/release/portmaster + + + RUN mkdir -p binary + COPY (+go-build/output/portmaster-core --GOARCH=amd64 --GOOS=linux --CMDS=portmaster-core) ./binary/portmaster-core + + COPY (+assets/assets.zip) ./binary/assets.zip + COPY (+angular-project/output/portmaster.zip --project=portmaster --dist=./dist --configuration=production --baseHref=/ui/modules/portmaster/) ./binary/portmaster.zip + + + # Download the intel data + RUN mkdir -p intel + + RUN wget -O ./intel/geoipv4.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv4_v20240529-0-1.mmdb.gz" && \ + wget -O ./intel/geoipv6.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv6_v20240529-0-1.mmdb.gz" && \ + gzip -d ./intel/geoipv4.mmdb.gz && \ + gzip -d ./intel/geoipv6.mmdb.gz + + RUN touch "./intel/index.dsd" + RUN touch "./intel/base.dsdl" + RUN touch "./intel/intermediate.dsdl" + RUN touch "./intel/urgent.dsdl" + + + # Generate index files + COPY (+go-build/output/updatemgr --GOARCH=amd64 --GOOS=linux --CMDS=updatemgr) ./updatemgr + RUN ./updatemgr -dir "./binary" -name "Binary" > ./binary/bin-index.json + RUN ./updatemgr -dir "./intel" -name "Intel" > ./intel/intel-index.json + + RUN cat ./binary/bin-index.json + RUN cat ./intel/intel-index.json + + # build the installers + RUN cargo tauri bundle --ci --target="${target}" + + # Installers + SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/deb/*.deb" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/" + SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/rpm/*.rpm" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/" kext-build: FROM ${rust_builder_image} diff --git a/cmds/updatemgr/confirm.go b/cmds/updatemgr/confirm.go deleted file mode 100644 index 293faaf6..00000000 --- a/cmds/updatemgr/confirm.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -import ( - "bufio" - "fmt" - "os" - "strings" -) - -func confirm(msg string) bool { - fmt.Printf("%s: [y|n] ", msg) - - scanner := bufio.NewScanner(os.Stdin) - ok := scanner.Scan() - if ok && strings.TrimSpace(scanner.Text()) == "y" { - return true - } - - return false -} diff --git a/cmds/updatemgr/main.go b/cmds/updatemgr/main.go index acd9a0d4..519d94eb 100644 --- a/cmds/updatemgr/main.go +++ b/cmds/updatemgr/main.go @@ -1,58 +1,51 @@ package main import ( + "encoding/json" + "flag" "fmt" "os" - "path/filepath" - "github.com/spf13/cobra" - - "github.com/safing/portmaster/base/updater" - "github.com/safing/portmaster/base/utils" + "github.com/safing/portmaster/service/updates" ) -var ( - registry *updater.ResourceRegistry - distDir string -) - -var rootCmd = &cobra.Command{ - Use: "updatemgr", - Short: "A simple tool to assist in the update and release process", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - // Check if the distribution directory exists. - absDistPath, err := filepath.Abs(distDir) - if err != nil { - return fmt.Errorf("failed to get absolute path of distribution directory: %w", err) - } - _, err = os.Stat(absDistPath) - if err != nil { - return fmt.Errorf("failed to access distribution directory: %w", err) - } - - registry = &updater.ResourceRegistry{} - err = registry.Initialize(utils.NewDirStructure(absDistPath, 0o0755)) - if err != nil { - return err - } - - err = registry.ScanStorage("") - if err != nil { - return err - } - - return nil +var binaryMap = map[string]updates.Artifact{ + "portmaster-core": { + Platform: "linux_amd64", + }, + "portmaster-core.exe": { + Platform: "windows_amd64", + }, + "portmaster-kext.sys": { + Platform: "windows_amd64", }, - SilenceUsage: true, -} - -func init() { - flags := rootCmd.PersistentFlags() - flags.StringVar(&distDir, "dist-dir", "dist", "Set the distribution directory. Falls back to ./dist if available.") } func main() { - if err := rootCmd.Execute(); err != nil { - os.Exit(1) + dir := flag.String("dir", "", "path to the directory that contains the artifacts") + name := flag.String("name", "", "name of the bundle") + version := flag.String("version", "", "version of the bundle") + + flag.Parse() + if *dir == "" { + fmt.Fprintf(os.Stderr, "-dir parameter is required\n") + return } + if *name == "" { + fmt.Fprintf(os.Stderr, "-name parameter is required\n") + return + } + + bundle, err := updates.GenerateBundleFromDir(*name, *version, binaryMap, *dir) + if err != nil { + fmt.Fprintf(os.Stderr, "failed to generate bundle: %s\n", err) + return + } + + bundleStr, err := json.MarshalIndent(&bundle, "", " ") + if err != nil { + fmt.Fprintf(os.Stderr, "failed to marshal bundle: %s\n", err) + } + + fmt.Printf("%s", bundleStr) } diff --git a/cmds/updatemgr/purge.go b/cmds/updatemgr/purge.go deleted file mode 100644 index d5b456ee..00000000 --- a/cmds/updatemgr/purge.go +++ /dev/null @@ -1,33 +0,0 @@ -package main - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/safing/portmaster/base/log" -) - -func init() { - rootCmd.AddCommand(purgeCmd) -} - -var purgeCmd = &cobra.Command{ - Use: "purge", - Short: "Remove old resource versions that are superseded by at least three versions", - RunE: purge, -} - -func purge(cmd *cobra.Command, args []string) error { - log.SetLogLevel(log.TraceLevel) - err := log.Start() - if err != nil { - fmt.Printf("failed to start logging: %s\n", err) - } - defer log.Shutdown() - - registry.SelectVersions() - registry.Purge(3) - - return nil -} diff --git a/cmds/updatemgr/release.go b/cmds/updatemgr/release.go deleted file mode 100644 index 0f5d596e..00000000 --- a/cmds/updatemgr/release.go +++ /dev/null @@ -1,195 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "os" - "path/filepath" - "time" - - "github.com/spf13/cobra" - - "github.com/safing/portmaster/base/updater" -) - -var ( - releaseCmd = &cobra.Command{ - Use: "release", - Short: "Release scans the distribution directory and creates registry indexes and the symlink structure", - Args: cobra.ExactArgs(1), - RunE: release, - } - preReleaseCmd = &cobra.Command{ - Use: "prerelease", - Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes", - Args: cobra.ExactArgs(1), - RunE: release, - } - preReleaseFrom string - resetPreReleases bool -) - -func init() { - rootCmd.AddCommand(releaseCmd) - rootCmd.AddCommand(preReleaseCmd) - - preReleaseCmd.Flags().StringVar(&preReleaseFrom, "from", "", "Make a pre-release based on the given channel") - _ = preReleaseCmd.MarkFlagRequired("from") - preReleaseCmd.Flags().BoolVar(&resetPreReleases, "reset", false, "Reset pre-release assets") -} - -func release(cmd *cobra.Command, args []string) error { - channel := args[0] - - // Check if we want to reset instead. - if resetPreReleases { - return removeFilesFromIndex(getChannelVersions(preReleaseFrom, true)) - } - - // Write new index. - err := writeIndex( - channel, - getChannelVersions(preReleaseFrom, false), - ) - if err != nil { - return err - } - - // Only when doing a release: - if preReleaseFrom == "" { - // Create symlinks to latest stable versions. - if !confirm("\nDo you want to write latest symlinks?") { - fmt.Println("aborted...") - return nil - } - symlinksDir := registry.StorageDir().ChildDir("latest", 0o755) - err = registry.CreateSymlinks(symlinksDir) - if err != nil { - return err - } - fmt.Println("written latest symlinks") - } - - return nil -} - -func writeIndex(channel string, versions map[string]string) error { - // Create new index file. - indexFile := &updater.IndexFile{ - Channel: channel, - Published: time.Now().UTC().Round(time.Second), - Releases: versions, - } - - // Export versions and format them. - confirmData, err := json.MarshalIndent(indexFile, "", " ") - if err != nil { - return err - } - - // Build index paths. - oldIndexPath := filepath.Join(registry.StorageDir().Path, channel+".json") - newIndexPath := filepath.Join(registry.StorageDir().Path, channel+".v2.json") - - // Print preview. - fmt.Printf("%s\n%s\n%s\n\n", channel, oldIndexPath, newIndexPath) - fmt.Println(string(confirmData)) - - // Ask for confirmation. - if !confirm("\nDo you want to write this index?") { - fmt.Println("aborted...") - return nil - } - - // Write indexes. - err = writeAsJSON(oldIndexPath, versions) - if err != nil { - return fmt.Errorf("failed to write %s: %w", oldIndexPath, err) - } - err = writeAsJSON(newIndexPath, indexFile) - if err != nil { - return fmt.Errorf("failed to write %s: %w", newIndexPath, err) - } - - return nil -} - -func writeAsJSON(path string, data any) error { - // Marshal to JSON. - jsonData, err := json.MarshalIndent(data, "", " ") - if err != nil { - return err - } - - // Write to disk. - err = os.WriteFile(path, jsonData, 0o0644) //nolint:gosec - if err != nil { - return err - } - - fmt.Printf("written %s\n", path) - return nil -} - -func removeFilesFromIndex(versions map[string]string) error { - // Print preview. - fmt.Println("To be deleted:") - for _, filePath := range versions { - fmt.Println(filePath) - } - - // Ask for confirmation. - if !confirm("\nDo you want to delete these files?") { - fmt.Println("aborted...") - return nil - } - - // Delete files. - for _, filePath := range versions { - err := os.Remove(filePath) - if err != nil { - return err - } - } - fmt.Println("deleted") - - return nil -} - -func getChannelVersions(prereleaseFrom string, storagePath bool) map[string]string { - if prereleaseFrom != "" { - registry.AddIndex(updater.Index{ - Path: prereleaseFrom + ".json", - PreRelease: false, - }) - err := registry.LoadIndexes(context.Background()) - if err != nil { - panic(err) - } - } - - // Sort all versions. - registry.SelectVersions() - export := registry.Export() - - // Go through all versions and save the highest version, if not stable or beta. - versions := make(map[string]string) - for _, rv := range export { - highestVersion := rv.Versions[0] - - // Ignore versions that are in the reference release channel. - if highestVersion.CurrentRelease { - continue - } - - // Add highest version of matching release channel. - if storagePath { - versions[rv.Identifier] = rv.GetFile().Path() - } else { - versions[rv.Identifier] = highestVersion.VersionNumber - } - } - - return versions -} diff --git a/cmds/updatemgr/scan.go b/cmds/updatemgr/scan.go deleted file mode 100644 index 674a581b..00000000 --- a/cmds/updatemgr/scan.go +++ /dev/null @@ -1,49 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - - "github.com/spf13/cobra" -) - -func init() { - rootCmd.AddCommand(scanCmd) -} - -var scanCmd = &cobra.Command{ - Use: "scan", - Short: "Scan the specified directory and print the result", - RunE: scan, -} - -func scan(cmd *cobra.Command, args []string) error { - // Reset and rescan. - registry.ResetResources() - err := registry.ScanStorage("") - if err != nil { - return err - } - - // Export latest versions. - data, err := json.MarshalIndent(exportSelected(true), "", " ") - if err != nil { - return err - } - // Print them. - fmt.Println(string(data)) - - return nil -} - -func exportSelected(preReleases bool) map[string]string { - registry.SetUsePreReleases(preReleases) - registry.SelectVersions() - export := registry.Export() - - versions := make(map[string]string) - for _, rv := range export { - versions[rv.Identifier] = rv.SelectedVersion.VersionNumber - } - return versions -} diff --git a/cmds/updatemgr/sign.go b/cmds/updatemgr/sign.go deleted file mode 100644 index 02cfb15d..00000000 --- a/cmds/updatemgr/sign.go +++ /dev/null @@ -1,303 +0,0 @@ -package main - -import ( - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - - "github.com/spf13/cobra" - - "github.com/safing/jess" - "github.com/safing/jess/filesig" - "github.com/safing/jess/truststores" -) - -func init() { - rootCmd.AddCommand(signCmd) - - // Required argument: envelope - signCmd.PersistentFlags().StringVarP(&envelopeName, "envelope", "", "", - "specify envelope name used for signing", - ) - _ = signCmd.MarkFlagRequired("envelope") - - // Optional arguments: verbose, tsdir, tskeyring - signCmd.PersistentFlags().BoolVarP(&signVerbose, "verbose", "v", false, - "enable verbose output", - ) - signCmd.PersistentFlags().StringVarP(&trustStoreDir, "tsdir", "", "", - "specify a truststore directory (default loaded from JESS_TS_DIR env variable)", - ) - signCmd.PersistentFlags().StringVarP(&trustStoreKeyring, "tskeyring", "", "", - "specify a truststore keyring namespace (default loaded from JESS_TS_KEYRING env variable) - lower priority than tsdir", - ) - - // Subcommand for signing indexes. - signCmd.AddCommand(signIndexCmd) -} - -var ( - signCmd = &cobra.Command{ - Use: "sign", - Short: "Sign resources", - RunE: sign, - Args: cobra.NoArgs, - } - signIndexCmd = &cobra.Command{ - Use: "index", - Short: "Sign indexes", - RunE: signIndex, - Args: cobra.ExactArgs(1), - } - - envelopeName string - signVerbose bool -) - -func sign(cmd *cobra.Command, args []string) error { - // Setup trust store. - trustStore, err := setupTrustStore() - if err != nil { - return err - } - - // Get envelope. - signingEnvelope, err := trustStore.GetEnvelope(envelopeName) - if err != nil { - return err - } - - // Get all resources and iterate over all versions. - export := registry.Export() - var verified, signed, fails int - for _, rv := range export { - for _, version := range rv.Versions { - file := version.GetFile() - - // Check if there is an existing signature. - _, err := os.Stat(file.Path() + filesig.Extension) - switch { - case err == nil || errors.Is(err, fs.ErrExist): - // If the file exists, just verify. - fileData, err := filesig.VerifyFile( - file.Path(), - file.Path()+filesig.Extension, - file.SigningMetadata(), - trustStore, - ) - if err != nil { - fmt.Printf("[FAIL] signature error for %s: %s\n", file.Path(), err) - fails++ - } else { - if signVerbose { - fmt.Printf("[ OK ] valid signature for %s: signed by %s\n", file.Path(), getSignedByMany(fileData, trustStore)) - } - verified++ - } - - case errors.Is(err, fs.ErrNotExist): - // Attempt to sign file. - fileData, err := filesig.SignFile( - file.Path(), - file.Path()+filesig.Extension, - file.SigningMetadata(), - signingEnvelope, - trustStore, - ) - if err != nil { - fmt.Printf("[FAIL] failed to sign %s: %s\n", file.Path(), err) - fails++ - } else { - fmt.Printf("[SIGN] signed %s with %s\n", file.Path(), getSignedBySingle(fileData, trustStore)) - signed++ - } - - default: - // File access error. - fmt.Printf("[FAIL] failed to access %s: %s\n", file.Path(), err) - fails++ - } - } - } - - if verified > 0 { - fmt.Printf("[STAT] verified %d files\n", verified) - } - if signed > 0 { - fmt.Printf("[STAT] signed %d files\n", signed) - } - if fails > 0 { - return fmt.Errorf("signing or verification failed on %d files", fails) - } - return nil -} - -func signIndex(cmd *cobra.Command, args []string) error { - // Setup trust store. - trustStore, err := setupTrustStore() - if err != nil { - return err - } - - // Get envelope. - signingEnvelope, err := trustStore.GetEnvelope(envelopeName) - if err != nil { - return err - } - - // Resolve globs. - files := make([]string, 0, len(args)) - for _, arg := range args { - matches, err := filepath.Glob(arg) - if err != nil { - return err - } - files = append(files, matches...) - } - - // Go through all files. - var verified, signed, fails int - for _, file := range files { - sigFile := file + filesig.Extension - - // Ignore matches for the signatures. - if strings.HasSuffix(file, filesig.Extension) { - continue - } - - // Check if there is an existing signature. - _, err := os.Stat(sigFile) - switch { - case err == nil || errors.Is(err, fs.ErrExist): - // If the file exists, just verify. - fileData, err := filesig.VerifyFile( - file, - sigFile, - nil, - trustStore, - ) - if err == nil { - if signVerbose { - fmt.Printf("[ OK ] valid signature for %s: signed by %s\n", file, getSignedByMany(fileData, trustStore)) - } - verified++ - - // Indexes are expected to change, so just sign the index again if verification fails. - continue - } - - fallthrough - case errors.Is(err, fs.ErrNotExist): - // Attempt to sign file. - fileData, err := filesig.SignFile( - file, - sigFile, - nil, - signingEnvelope, - trustStore, - ) - if err != nil { - fmt.Printf("[FAIL] failed to sign %s: %s\n", file, err) - fails++ - } else { - fmt.Printf("[SIGN] signed %s with %s\n", file, getSignedBySingle(fileData, trustStore)) - signed++ - } - - default: - // File access error. - fmt.Printf("[FAIL] failed to access %s: %s\n", sigFile, err) - fails++ - } - } - - if verified > 0 { - fmt.Printf("[STAT] verified %d files", verified) - } - if signed > 0 { - fmt.Printf("[STAT] signed %d files", signed) - } - if fails > 0 { - return fmt.Errorf("signing failed on %d files", fails) - } - return nil -} - -var ( - trustStoreDir string - trustStoreKeyring string -) - -func setupTrustStore() (trustStore truststores.ExtendedTrustStore, err error) { - // Get trust store directory. - if trustStoreDir == "" { - trustStoreDir, _ = os.LookupEnv("JESS_TS_DIR") - if trustStoreDir == "" { - trustStoreDir, _ = os.LookupEnv("JESS_TSDIR") - } - } - if trustStoreDir != "" { - trustStore, err = truststores.NewDirTrustStore(trustStoreDir) - if err != nil { - return nil, err - } - } - - // Get trust store keyring. - if trustStore == nil { - if trustStoreKeyring == "" { - trustStoreKeyring, _ = os.LookupEnv("JESS_TS_KEYRING") - if trustStoreKeyring == "" { - trustStoreKeyring, _ = os.LookupEnv("JESS_TSKEYRING") - } - } - if trustStoreKeyring != "" { - trustStore, err = truststores.NewKeyringTrustStore(trustStoreKeyring) - if err != nil { - return nil, err - } - } - } - - // Truststore is mandatory. - if trustStore == nil { - return nil, errors.New("no truststore configured, please pass arguments or use env variables") - } - - return trustStore, nil -} - -func getSignedByMany(fds []*filesig.FileData, trustStore jess.TrustStore) string { - signedBy := make([]string, 0, len(fds)) - for _, fd := range fds { - if sig := fd.Signature(); sig != nil { - for _, seal := range sig.Signatures { - if signet, err := trustStore.GetSignet(seal.ID, true); err == nil { - signedBy = append(signedBy, fmt.Sprintf("%s (%s)", signet.Info.Name, seal.ID)) - } else { - signedBy = append(signedBy, seal.ID) - } - } - } - } - return strings.Join(signedBy, " and ") -} - -func getSignedBySingle(fd *filesig.FileData, trustStore jess.TrustStore) string { - if sig := fd.Signature(); sig != nil { - signedBy := make([]string, 0, len(sig.Signatures)) - for _, seal := range sig.Signatures { - if signet, err := trustStore.GetSignet(seal.ID, true); err == nil { - signedBy = append(signedBy, fmt.Sprintf("%s (%s)", signet.Info.Name, seal.ID)) - } else { - signedBy = append(signedBy, seal.ID) - } - } - return strings.Join(signedBy, " and ") - } - - return "" -} diff --git a/desktop/tauri/src-tauri/tauri.conf.json5 b/desktop/tauri/src-tauri/tauri.conf.json5 index 2d9de0d7..5bd85972 100644 --- a/desktop/tauri/src-tauri/tauri.conf.json5 +++ b/desktop/tauri/src-tauri/tauri.conf.json5 @@ -59,18 +59,25 @@ ], "desktopTemplate": "../../../packaging/linux/portmaster.desktop", "files": { + // Service file "/usr/lib/systemd/system/portmaster.service": "../../../packaging/linux/portmaster.service", - "/usr/lib/portmaster/bin-index.json": "binaries/bin-index.json", - "/usr/lib/portmaster/portmaster-core": "binaries/portmaster-core", - "/usr/lib/portmaster/portmaster.zip": "binaries/portmaster.zip", - "/usr/lib/portmaster/assets.zip": "binaries/assets.zip", - "/var/lib/portmaster/intel/intel-index.json": "binaries/intel-index.json", - "/var/lib/portmaster/intel/base.dsdl": "binaries/base.dsdl", - "/var/lib/portmaster/intel/geoipv4.mmdb": "binaries/geoipv4.mmdb", - "/var/lib/portmaster/intel/geoipv6.mmdb": "binaries/geoipv6.mmdb", - "/var/lib/portmaster/intel/index.dsd": "binaries/index.dsd", - "/var/lib/portmaster/intel/intermediate.dsdl": "binaries/intermediate.dsdl", - "/var/lib/portmaster/intel/urgent.dsdl": "binaries/urgent.dsdl", + + // Binary files + "/usr/lib/portmaster/bin-index.json": "binary/bin-index.json", + "/usr/lib/portmaster/portmaster-core": "binary/portmaster-core", + "/usr/lib/portmaster/portmaster.zip": "binary/portmaster.zip", + "/usr/lib/portmaster/assets.zip": "binary/assets.zip", + + // Intel files + "/var/lib/portmaster/intel/intel-index.json": "intel/intel-index.json", + "/var/lib/portmaster/intel/base.dsdl": "intel/base.dsdl", + "/var/lib/portmaster/intel/geoipv4.mmdb": "intel/geoipv4.mmdb", + "/var/lib/portmaster/intel/geoipv6.mmdb": "intel/geoipv6.mmdb", + "/var/lib/portmaster/intel/index.dsd": "intel/index.dsd", + "/var/lib/portmaster/intel/intermediate.dsdl": "intel/intermediate.dsdl", + "/var/lib/portmaster/intel/urgent.dsdl": "intel/urgent.dsdl", + + // Shortcut "/etc/xdg/autostart/portmaster.desktop": "../../../packaging/linux/portmaster-autostart.desktop" }, "postInstallScript": "../../../packaging/linux/postinst", @@ -83,18 +90,25 @@ "desktopTemplate": "../../../packaging/linux/portmaster.desktop", "release": "1", "files": { + // Service file "/usr/lib/systemd/system/portmaster.service": "../../../packaging/linux/portmaster.service", - "/usr/lib/portmaster/bin-index.json": "binaries/bin-index.json", - "/usr/lib/portmaster/portmaster-core": "binaries/portmaster-core", - "/usr/lib/portmaster/portmaster.zip": "binaries/portmaster.zip", - "/usr/lib/portmaster/assets.zip": "binaries/assets.zip", - "/var/lib/portmaster/intel/intel-index.json": "binaries/intel-index.json", - "/var/lib/portmaster/intel/base.dsdl": "binaries/base.dsdl", - "/var/lib/portmaster/intel/geoipv4.mmdb": "binaries/geoipv4.mmdb", - "/var/lib/portmaster/intel/geoipv6.mmdb": "binaries/geoipv6.mmdb", - "/var/lib/portmaster/intel/index.dsd": "binaries/index.dsd", - "/var/lib/portmaster/intel/intermediate.dsdl": "binaries/intermediate.dsdl", - "/var/lib/portmaster/intel/urgent.dsdl": "binaries/urgent.dsdl", + + // Binary files + "/usr/lib/portmaster/bin-index.json": "binary/bin-index.json", + "/usr/lib/portmaster/portmaster-core": "binary/portmaster-core", + "/usr/lib/portmaster/portmaster.zip": "binary/portmaster.zip", + "/usr/lib/portmaster/assets.zip": "binary/assets.zip", + + // Intel files + "/var/lib/portmaster/intel/intel-index.json": "intel/intel-index.json", + "/var/lib/portmaster/intel/base.dsdl": "intel/base.dsdl", + "/var/lib/portmaster/intel/geoipv4.mmdb": "intel/geoipv4.mmdb", + "/var/lib/portmaster/intel/geoipv6.mmdb": "intel/geoipv6.mmdb", + "/var/lib/portmaster/intel/index.dsd": "intel/index.dsd", + "/var/lib/portmaster/intel/intermediate.dsdl": "intel/intermediate.dsdl", + "/var/lib/portmaster/intel/urgent.dsdl": "intel/urgent.dsdl", + + // Shortcut "/etc/xdg/autostart/portmaster.desktop": "../../../packaging/linux/portmaster-autostart.desktop" }, "postInstallScript": "../../../packaging/linux/postinst", diff --git a/service/updates/bundle.go b/service/updates/bundle.go index 02b6089c..6cd2d430 100644 --- a/service/updates/bundle.go +++ b/service/updates/bundle.go @@ -120,3 +120,57 @@ func checkIfFileIsValid(filename string, artifact Artifact) (bool, error) { } return true, nil } + +// GenerateBundleFromDir generates a bundle from a given folder. +func GenerateBundleFromDir(name string, version string, properties map[string]Artifact, dirPath string) (*Bundle, error) { + files, err := os.ReadDir(dirPath) + if err != nil { + return nil, err + } + artifacts := make([]Artifact, 0, len(files)) + for _, f := range files { + // Skip dirs + if f.IsDir() { + continue + } + + artifact := Artifact{ + Filename: f.Name(), + } + predefined, ok := properties[f.Name()] + // Check if caller supplied predefined settings for this artifact. + if ok { + // File that have compression may have different filename and artifact filename. (because of the extension) + // If caller did not specify the artifact filename set it as the same as the filename. + if predefined.Filename == "" { + predefined.Filename = f.Name() + } + artifact = predefined + } + content, err := os.ReadFile(filepath.Join(dirPath, f.Name())) + if err != nil { + return nil, err + } + + // Decompress if compression was applied to the file. + if artifact.Unpack != "" { + content, err = unpack(artifact.Unpack, content) + if err != nil { + return nil, err + } + } + + hash := sha256.Sum256(content) + hashStr := hex.EncodeToString(hash[:]) + artifact.SHA256 = hashStr + + artifacts = append(artifacts, artifact) + } + + return &Bundle{ + Name: name, + Version: version, + Artifacts: artifacts, + Published: time.Now(), + }, nil +}