[WIP] Add CI for building deb,rpm installers

This commit is contained in:
Vladimir Stoilov 2024-10-02 15:37:39 +03:00
parent 3411e08500
commit 8e1f3c0ed9
No known key found for this signature in database
GPG key ID: 2F190B67A43A81AF
9 changed files with 212 additions and 814 deletions

233
Earthfile
View file

@ -3,6 +3,7 @@ VERSION --arg-scope-and-set --global-cache 0.8
ARG --global go_version = 1.22
ARG --global node_version = 18
ARG --global rust_version = 1.79
ARG --global tauri_version = "2.0.0-rc.8"
ARG --global golangci_lint_version = 1.57.1
ARG --global go_builder_image = "golang:${go_version}-alpine"
@ -56,15 +57,14 @@ build:
# Build Tauri app binaries:
# ./dist/linux_amd64/portmaster-app
# ./dist/windows_amd64/portmaster-app
BUILD +tauri-build --target="x86_64-unknown-linux-gnu"
BUILD +tauri-build --target="x86_64-pc-windows-gnu"
# TODO(vladimir): Build bundles
# ./dist/linux_amd64/Portmaster-0.1.0-1.x86_64.rpm
# ./dist/linux_amd64/Portmaster_0.1.0_amd64.deb
BUILD +tauri-build --target="x86_64-unknown-linux-gnu"
# TODO:
# BUILD +tauri-build --target="x86_64-pc-windows-gnu"
# Bild Tauri bundle for Windows:
# ./dist/windows_amd64/portmaster-app_vX-X-X.zip
BUILD +tauri-build-windows-bundle
# Build UI assets:
# ./dist/all/assets.zip
@ -82,7 +82,7 @@ angular-ci:
tauri-ci:
BUILD +tauri-build --target="x86_64-unknown-linux-gnu"
BUILD +tauri-build-windows-bundle
BUILD +tauri-build --target="x86_64-pc-windows-gnu"
kext-ci:
BUILD +kext-build
@ -349,6 +349,7 @@ angular-project:
# Save portmaster UI as local artifact.
IF [ "${project}" = "portmaster" ]
SAVE ARTIFACT --keep-ts "./${project}.zip" AS LOCAL ${outputDir}/all/${project}-ui.zip
SAVE ARTIFACT --keep-ts "./${project}.zip" output/${project}.zip
END
# Build the angular projects (portmaster-UI and tauri-builtin) in dev mode
@ -420,7 +421,7 @@ rust-base:
DO rust+INIT --keep_fingerprints=true
# For now we need tauri-cli 2.0.0 for bulding
DO rust+CARGO --args="install tauri-cli --version ^2.0.0-beta"
DO rust+CARGO --args="install tauri-cli --version ${tauri_version} --locked"
# Explicitly cache here.
SAVE IMAGE --cache-hint
@ -434,7 +435,7 @@ tauri-src:
# are preserved such that Rust's incremental compilation works correctly.
COPY --keep-ts ./desktop/tauri/ .
COPY assets/data ./../../assets/data
COPY packaging/linux ./../../packaging/linux
COPY packaging ./../../packaging
COPY (+angular-project/output/tauri-builtin --project=tauri-builtin --dist=./dist/tauri-builtin --configuration=production --baseHref="/") ./../angular/dist/tauri-builtin
WORKDIR /app/tauri/src-tauri
@ -446,31 +447,14 @@ tauri-build:
FROM +tauri-src
ARG --required target
ARG output=".*/release/(([^\./]+|([^\./]+\.(dll|exe)))|bundle/(deb|rpm)/.*\.(deb|rpm))"
ARG bundle="none"
# if we want tauri to create the installer bundles we also need to provide all external binaries
# we need to do some magic here because tauri expects the binaries to include the rust target tripple.
# We already know that triple because it's a required argument. From that triple, we use +RUST_TO_GO_ARCH_STRING
# function from below to parse the triple and guess wich GOOS and GOARCH we need.
RUN mkdir /tmp/gobuild
RUN mkdir ./binaries
ARG output=".*/release/([^\./]+|([^\./]+\.(dll|exe)))"
DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
# Just for debugging ...
# RUN ls -R ./binaries
# The following is exected to work but doesn't. for whatever reason cargo-sweep errors out on the windows-toolchain.
#
# DO rust+CARGO --args="tauri build --bundles none --ci --target=${target}" --output="release/[^/\.]+"
#
# For, now, we just directly mount the rust target cache and call cargo ourself.
DO rust+SET_CACHE_MOUNTS_ENV
RUN rustup target add "${target}"
RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --ci --target="${target}"
RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --ci --target="${target}" --no-bundle
DO rust+COPY_OUTPUT --output="${output}"
# BUG(cross-compilation):
@ -489,127 +473,13 @@ tauri-build:
RUN echo output: $(ls -R "target/${target}/release")
# Binaries
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster-app"
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster-app.exe"
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/WebView2Loader.dll" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/WebView2Loader.dll"
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster"
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/portmaster.exe"
# SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/WebView2Loader.dll" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/WebView2Loader.dll"
# Installers
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/deb/*.deb" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/rpm/*.rpm" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster" ./output/portmaster
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/portmaster.exe" ./output/portmaster.exe
tauri-build-windows-bundle:
FROM +tauri-src
ARG target="x86_64-pc-windows-gnu"
ARG output=".*/release/(([^\./]+|([^\./]+\.(dll|exe))))"
ARG bundle="none"
ARG GOOS=windows
ARG GOARCH=amd64
ARG GOARM
# The binaries will not be used but we still need to create them. Tauri will check for them.
RUN mkdir /tmp/gobuild
RUN mkdir ./binaries
DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
# Our tauri app has externalBins configured so tauri will look for them when it finished compiling
# the app. Make sure we copy portmaster-start and portmaster-core in all architectures supported.
# See documentation for externalBins for more information on how tauri searches for the binaries.
COPY (+go-build/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild
# Place them in the correct folder with the rust target tripple attached.
FOR bin IN $(ls /tmp/gobuild)
# ${bin$.*} does not work in SET commands unfortunately so we use a shell
# snippet here:
RUN set -e ; \
dest="./binaries/${bin}-${target}" ; \
if [ -z "${bin##*.exe}" ]; then \
dest="./binaries/${bin%.*}-${target}.exe" ; \
fi ; \
cp "/tmp/gobuild/${bin}" "${dest}" ;
END
# Just for debugging ...
# RUN ls -R ./binaries
DO rust+SET_CACHE_MOUNTS_ENV
RUN rustup target add "${target}"
RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --no-bundle --ci --target="${target}"
DO rust+COPY_OUTPUT --output="${output}"
# Get version from git.
COPY .git .
LET version = "$(git tag --points-at || true)"
IF [ -z "${version}" ]
LET dev_version = "$(git describe --tags --first-parent --abbrev=0 || true)"
IF [ -n "${dev_version}" ]
SET version = "${dev_version}"
END
END
IF [ -z "${version}" ]
SET version = "v0.0.0"
END
ENV VERSION="${version}"
RUN echo "Version: $VERSION"
ENV VERSION_SUFFIX="$(echo $VERSION | tr '.' '-')"
RUN echo "Version Suffix: $VERSION_SUFFIX"
RUN echo output: $(ls -R "target/${target}/release")
RUN mv "target/${target}/release/portmaster.exe" "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.exe"
RUN zip "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.zip" "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.exe" -j portmaster-app${VERSION_SUFFIX}.exe "target/${target}/release/WebView2Loader.dll" -j WebView2Loader.dll
SAVE ARTIFACT --if-exists "target/${target}/release/portmaster-app_${VERSION_SUFFIX}.zip" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
tauri-prep-windows:
FROM +angular-base --configuration=production
ARG target="x86_64-pc-windows-msvc"
# if we want tauri to create the installer bundles we also need to provide all external binaries
# we need to do some magic here because tauri expects the binaries to include the rust target tripple.
# We already know that triple because it's a required argument. From that triple, we use +RUST_TO_GO_ARCH_STRING
# function from below to parse the triple and guess wich GOOS and GOARCH we need.
RUN mkdir /tmp/gobuild
RUN mkdir ./binaries
DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
# Our tauri app has externalBins configured so tauri will try to embed them when it finished compiling
# the app. Make sure we copy portmaster-start and portmaster-core in all architectures supported.
# See documentation for externalBins for more information on how tauri searches for the binaries.
COPY (+go-build/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild
# Place them in the correct folder with the rust target tripple attached.
FOR bin IN $(ls /tmp/gobuild)
# ${bin$.*} does not work in SET commands unfortunately so we use a shell
# snippet here:
RUN set -e ; \
dest="./binaries/${bin}-${target}" ; \
if [ -z "${bin##*.exe}" ]; then \
dest="./binaries/${bin%.*}-${target}.exe" ; \
fi ; \
cp "/tmp/gobuild/${bin}" "${dest}" ;
END
# Copy source
COPY --keep-ts ./desktop/tauri/src-tauri src-tauri
COPY --keep-ts ./assets assets
# Build UI
ENV NODE_ENV="production"
RUN --no-cache ./node_modules/.bin/ng build --configuration production --base-href / "tauri-builtin"
# Just for debugging ...
# RUN ls -R ./binaries
# RUN ls -R ./dist
SAVE ARTIFACT "./dist/tauri-builtin" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/angular/dist/"
SAVE ARTIFACT "./src-tauri" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/tauri/src-tauri"
SAVE ARTIFACT "./binaries" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/desktop/tauri/src-tauri/"
SAVE ARTIFACT "./assets" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/assets"
tauri-release:
FROM ${work_image}
@ -624,11 +494,78 @@ tauri-lint:
ARG target="x86_64-unknown-linux-gnu"
WORKDIR /app
# Copy static files that are embedded inside the executable.
COPY --keep-ts ./assets ./assets
# Copy all the rust code
COPY --keep-ts ./desktop/tauri ./desktop/tauri
# Create a empty ui dir so it will satisfy the build.
RUN mkdir -p ./desktop/angular/dist/tauri-builtin
SAVE IMAGE --cache-hint
# Run the linter.
WORKDIR /app/desktop/tauri/src-tauri
RUN cargo clippy --all-targets --all-features -- -D warnings
RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo clippy --all-targets --all-features -- -D warnings
tauri-bundle-linux:
FROM +rust-base
# ARG --required target
ARG target="x86_64-unknown-linux-gnu"
WORKDIR /app/tauri
COPY --keep-ts ./desktop/tauri/ .
COPY assets/data ./../../assets/data
COPY packaging ./../../packaging
WORKDIR /app/tauri/src-tauri
SAVE IMAGE --cache-hint
DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
# Build and copy the binaries
RUN mkdir -p target/${target}/release
COPY (+tauri-build/output/portmaster --target=x86_64-unknown-linux-gnu) ./target/${target}/release/portmaster
RUN mkdir -p binary
COPY (+go-build/output/portmaster-core --GOARCH=amd64 --GOOS=linux --CMDS=portmaster-core) ./binary/portmaster-core
COPY (+assets/assets.zip) ./binary/assets.zip
COPY (+angular-project/output/portmaster.zip --project=portmaster --dist=./dist --configuration=production --baseHref=/ui/modules/portmaster/) ./binary/portmaster.zip
# Download the intel data
RUN mkdir -p intel
RUN wget -O ./intel/geoipv4.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv4_v20240529-0-1.mmdb.gz" && \
wget -O ./intel/geoipv6.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv6_v20240529-0-1.mmdb.gz" && \
gzip -d ./intel/geoipv4.mmdb.gz && \
gzip -d ./intel/geoipv6.mmdb.gz
RUN touch "./intel/index.dsd"
RUN touch "./intel/base.dsdl"
RUN touch "./intel/intermediate.dsdl"
RUN touch "./intel/urgent.dsdl"
# Generate index files
COPY (+go-build/output/updatemgr --GOARCH=amd64 --GOOS=linux --CMDS=updatemgr) ./updatemgr
RUN ./updatemgr -dir "./binary" -name "Binary" > ./binary/bin-index.json
RUN ./updatemgr -dir "./intel" -name "Intel" > ./intel/intel-index.json
RUN cat ./binary/bin-index.json
RUN cat ./intel/intel-index.json
# build the installers
RUN cargo tauri bundle --ci --target="${target}"
# Installers
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/deb/*.deb" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
SAVE ARTIFACT --if-exists --keep-ts "target/${target}/release/bundle/rpm/*.rpm" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/"
kext-build:
FROM ${rust_builder_image}

View file

@ -1,20 +0,0 @@
package main
import (
"bufio"
"fmt"
"os"
"strings"
)
func confirm(msg string) bool {
fmt.Printf("%s: [y|n] ", msg)
scanner := bufio.NewScanner(os.Stdin)
ok := scanner.Scan()
if ok && strings.TrimSpace(scanner.Text()) == "y" {
return true
}
return false
}

View file

@ -1,58 +1,51 @@
package main
import (
"encoding/json"
"flag"
"fmt"
"os"
"path/filepath"
"github.com/spf13/cobra"
"github.com/safing/portmaster/base/updater"
"github.com/safing/portmaster/base/utils"
"github.com/safing/portmaster/service/updates"
)
var (
registry *updater.ResourceRegistry
distDir string
)
var rootCmd = &cobra.Command{
Use: "updatemgr",
Short: "A simple tool to assist in the update and release process",
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
// Check if the distribution directory exists.
absDistPath, err := filepath.Abs(distDir)
if err != nil {
return fmt.Errorf("failed to get absolute path of distribution directory: %w", err)
}
_, err = os.Stat(absDistPath)
if err != nil {
return fmt.Errorf("failed to access distribution directory: %w", err)
}
registry = &updater.ResourceRegistry{}
err = registry.Initialize(utils.NewDirStructure(absDistPath, 0o0755))
if err != nil {
return err
}
err = registry.ScanStorage("")
if err != nil {
return err
}
return nil
var binaryMap = map[string]updates.Artifact{
"portmaster-core": {
Platform: "linux_amd64",
},
"portmaster-core.exe": {
Platform: "windows_amd64",
},
"portmaster-kext.sys": {
Platform: "windows_amd64",
},
SilenceUsage: true,
}
func init() {
flags := rootCmd.PersistentFlags()
flags.StringVar(&distDir, "dist-dir", "dist", "Set the distribution directory. Falls back to ./dist if available.")
}
func main() {
if err := rootCmd.Execute(); err != nil {
os.Exit(1)
dir := flag.String("dir", "", "path to the directory that contains the artifacts")
name := flag.String("name", "", "name of the bundle")
version := flag.String("version", "", "version of the bundle")
flag.Parse()
if *dir == "" {
fmt.Fprintf(os.Stderr, "-dir parameter is required\n")
return
}
if *name == "" {
fmt.Fprintf(os.Stderr, "-name parameter is required\n")
return
}
bundle, err := updates.GenerateBundleFromDir(*name, *version, binaryMap, *dir)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to generate bundle: %s\n", err)
return
}
bundleStr, err := json.MarshalIndent(&bundle, "", " ")
if err != nil {
fmt.Fprintf(os.Stderr, "failed to marshal bundle: %s\n", err)
}
fmt.Printf("%s", bundleStr)
}

View file

@ -1,33 +0,0 @@
package main
import (
"fmt"
"github.com/spf13/cobra"
"github.com/safing/portmaster/base/log"
)
func init() {
rootCmd.AddCommand(purgeCmd)
}
var purgeCmd = &cobra.Command{
Use: "purge",
Short: "Remove old resource versions that are superseded by at least three versions",
RunE: purge,
}
func purge(cmd *cobra.Command, args []string) error {
log.SetLogLevel(log.TraceLevel)
err := log.Start()
if err != nil {
fmt.Printf("failed to start logging: %s\n", err)
}
defer log.Shutdown()
registry.SelectVersions()
registry.Purge(3)
return nil
}

View file

@ -1,195 +0,0 @@
package main
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"time"
"github.com/spf13/cobra"
"github.com/safing/portmaster/base/updater"
)
var (
releaseCmd = &cobra.Command{
Use: "release",
Short: "Release scans the distribution directory and creates registry indexes and the symlink structure",
Args: cobra.ExactArgs(1),
RunE: release,
}
preReleaseCmd = &cobra.Command{
Use: "prerelease",
Short: "Stage scans the specified directory and loads the indexes - it then creates a staging index with all files newer than the stable and beta indexes",
Args: cobra.ExactArgs(1),
RunE: release,
}
preReleaseFrom string
resetPreReleases bool
)
func init() {
rootCmd.AddCommand(releaseCmd)
rootCmd.AddCommand(preReleaseCmd)
preReleaseCmd.Flags().StringVar(&preReleaseFrom, "from", "", "Make a pre-release based on the given channel")
_ = preReleaseCmd.MarkFlagRequired("from")
preReleaseCmd.Flags().BoolVar(&resetPreReleases, "reset", false, "Reset pre-release assets")
}
func release(cmd *cobra.Command, args []string) error {
channel := args[0]
// Check if we want to reset instead.
if resetPreReleases {
return removeFilesFromIndex(getChannelVersions(preReleaseFrom, true))
}
// Write new index.
err := writeIndex(
channel,
getChannelVersions(preReleaseFrom, false),
)
if err != nil {
return err
}
// Only when doing a release:
if preReleaseFrom == "" {
// Create symlinks to latest stable versions.
if !confirm("\nDo you want to write latest symlinks?") {
fmt.Println("aborted...")
return nil
}
symlinksDir := registry.StorageDir().ChildDir("latest", 0o755)
err = registry.CreateSymlinks(symlinksDir)
if err != nil {
return err
}
fmt.Println("written latest symlinks")
}
return nil
}
func writeIndex(channel string, versions map[string]string) error {
// Create new index file.
indexFile := &updater.IndexFile{
Channel: channel,
Published: time.Now().UTC().Round(time.Second),
Releases: versions,
}
// Export versions and format them.
confirmData, err := json.MarshalIndent(indexFile, "", " ")
if err != nil {
return err
}
// Build index paths.
oldIndexPath := filepath.Join(registry.StorageDir().Path, channel+".json")
newIndexPath := filepath.Join(registry.StorageDir().Path, channel+".v2.json")
// Print preview.
fmt.Printf("%s\n%s\n%s\n\n", channel, oldIndexPath, newIndexPath)
fmt.Println(string(confirmData))
// Ask for confirmation.
if !confirm("\nDo you want to write this index?") {
fmt.Println("aborted...")
return nil
}
// Write indexes.
err = writeAsJSON(oldIndexPath, versions)
if err != nil {
return fmt.Errorf("failed to write %s: %w", oldIndexPath, err)
}
err = writeAsJSON(newIndexPath, indexFile)
if err != nil {
return fmt.Errorf("failed to write %s: %w", newIndexPath, err)
}
return nil
}
func writeAsJSON(path string, data any) error {
// Marshal to JSON.
jsonData, err := json.MarshalIndent(data, "", " ")
if err != nil {
return err
}
// Write to disk.
err = os.WriteFile(path, jsonData, 0o0644) //nolint:gosec
if err != nil {
return err
}
fmt.Printf("written %s\n", path)
return nil
}
func removeFilesFromIndex(versions map[string]string) error {
// Print preview.
fmt.Println("To be deleted:")
for _, filePath := range versions {
fmt.Println(filePath)
}
// Ask for confirmation.
if !confirm("\nDo you want to delete these files?") {
fmt.Println("aborted...")
return nil
}
// Delete files.
for _, filePath := range versions {
err := os.Remove(filePath)
if err != nil {
return err
}
}
fmt.Println("deleted")
return nil
}
func getChannelVersions(prereleaseFrom string, storagePath bool) map[string]string {
if prereleaseFrom != "" {
registry.AddIndex(updater.Index{
Path: prereleaseFrom + ".json",
PreRelease: false,
})
err := registry.LoadIndexes(context.Background())
if err != nil {
panic(err)
}
}
// Sort all versions.
registry.SelectVersions()
export := registry.Export()
// Go through all versions and save the highest version, if not stable or beta.
versions := make(map[string]string)
for _, rv := range export {
highestVersion := rv.Versions[0]
// Ignore versions that are in the reference release channel.
if highestVersion.CurrentRelease {
continue
}
// Add highest version of matching release channel.
if storagePath {
versions[rv.Identifier] = rv.GetFile().Path()
} else {
versions[rv.Identifier] = highestVersion.VersionNumber
}
}
return versions
}

View file

@ -1,49 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"github.com/spf13/cobra"
)
func init() {
rootCmd.AddCommand(scanCmd)
}
var scanCmd = &cobra.Command{
Use: "scan",
Short: "Scan the specified directory and print the result",
RunE: scan,
}
func scan(cmd *cobra.Command, args []string) error {
// Reset and rescan.
registry.ResetResources()
err := registry.ScanStorage("")
if err != nil {
return err
}
// Export latest versions.
data, err := json.MarshalIndent(exportSelected(true), "", " ")
if err != nil {
return err
}
// Print them.
fmt.Println(string(data))
return nil
}
func exportSelected(preReleases bool) map[string]string {
registry.SetUsePreReleases(preReleases)
registry.SelectVersions()
export := registry.Export()
versions := make(map[string]string)
for _, rv := range export {
versions[rv.Identifier] = rv.SelectedVersion.VersionNumber
}
return versions
}

View file

@ -1,303 +0,0 @@
package main
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"github.com/safing/jess"
"github.com/safing/jess/filesig"
"github.com/safing/jess/truststores"
)
func init() {
rootCmd.AddCommand(signCmd)
// Required argument: envelope
signCmd.PersistentFlags().StringVarP(&envelopeName, "envelope", "", "",
"specify envelope name used for signing",
)
_ = signCmd.MarkFlagRequired("envelope")
// Optional arguments: verbose, tsdir, tskeyring
signCmd.PersistentFlags().BoolVarP(&signVerbose, "verbose", "v", false,
"enable verbose output",
)
signCmd.PersistentFlags().StringVarP(&trustStoreDir, "tsdir", "", "",
"specify a truststore directory (default loaded from JESS_TS_DIR env variable)",
)
signCmd.PersistentFlags().StringVarP(&trustStoreKeyring, "tskeyring", "", "",
"specify a truststore keyring namespace (default loaded from JESS_TS_KEYRING env variable) - lower priority than tsdir",
)
// Subcommand for signing indexes.
signCmd.AddCommand(signIndexCmd)
}
var (
signCmd = &cobra.Command{
Use: "sign",
Short: "Sign resources",
RunE: sign,
Args: cobra.NoArgs,
}
signIndexCmd = &cobra.Command{
Use: "index",
Short: "Sign indexes",
RunE: signIndex,
Args: cobra.ExactArgs(1),
}
envelopeName string
signVerbose bool
)
func sign(cmd *cobra.Command, args []string) error {
// Setup trust store.
trustStore, err := setupTrustStore()
if err != nil {
return err
}
// Get envelope.
signingEnvelope, err := trustStore.GetEnvelope(envelopeName)
if err != nil {
return err
}
// Get all resources and iterate over all versions.
export := registry.Export()
var verified, signed, fails int
for _, rv := range export {
for _, version := range rv.Versions {
file := version.GetFile()
// Check if there is an existing signature.
_, err := os.Stat(file.Path() + filesig.Extension)
switch {
case err == nil || errors.Is(err, fs.ErrExist):
// If the file exists, just verify.
fileData, err := filesig.VerifyFile(
file.Path(),
file.Path()+filesig.Extension,
file.SigningMetadata(),
trustStore,
)
if err != nil {
fmt.Printf("[FAIL] signature error for %s: %s\n", file.Path(), err)
fails++
} else {
if signVerbose {
fmt.Printf("[ OK ] valid signature for %s: signed by %s\n", file.Path(), getSignedByMany(fileData, trustStore))
}
verified++
}
case errors.Is(err, fs.ErrNotExist):
// Attempt to sign file.
fileData, err := filesig.SignFile(
file.Path(),
file.Path()+filesig.Extension,
file.SigningMetadata(),
signingEnvelope,
trustStore,
)
if err != nil {
fmt.Printf("[FAIL] failed to sign %s: %s\n", file.Path(), err)
fails++
} else {
fmt.Printf("[SIGN] signed %s with %s\n", file.Path(), getSignedBySingle(fileData, trustStore))
signed++
}
default:
// File access error.
fmt.Printf("[FAIL] failed to access %s: %s\n", file.Path(), err)
fails++
}
}
}
if verified > 0 {
fmt.Printf("[STAT] verified %d files\n", verified)
}
if signed > 0 {
fmt.Printf("[STAT] signed %d files\n", signed)
}
if fails > 0 {
return fmt.Errorf("signing or verification failed on %d files", fails)
}
return nil
}
func signIndex(cmd *cobra.Command, args []string) error {
// Setup trust store.
trustStore, err := setupTrustStore()
if err != nil {
return err
}
// Get envelope.
signingEnvelope, err := trustStore.GetEnvelope(envelopeName)
if err != nil {
return err
}
// Resolve globs.
files := make([]string, 0, len(args))
for _, arg := range args {
matches, err := filepath.Glob(arg)
if err != nil {
return err
}
files = append(files, matches...)
}
// Go through all files.
var verified, signed, fails int
for _, file := range files {
sigFile := file + filesig.Extension
// Ignore matches for the signatures.
if strings.HasSuffix(file, filesig.Extension) {
continue
}
// Check if there is an existing signature.
_, err := os.Stat(sigFile)
switch {
case err == nil || errors.Is(err, fs.ErrExist):
// If the file exists, just verify.
fileData, err := filesig.VerifyFile(
file,
sigFile,
nil,
trustStore,
)
if err == nil {
if signVerbose {
fmt.Printf("[ OK ] valid signature for %s: signed by %s\n", file, getSignedByMany(fileData, trustStore))
}
verified++
// Indexes are expected to change, so just sign the index again if verification fails.
continue
}
fallthrough
case errors.Is(err, fs.ErrNotExist):
// Attempt to sign file.
fileData, err := filesig.SignFile(
file,
sigFile,
nil,
signingEnvelope,
trustStore,
)
if err != nil {
fmt.Printf("[FAIL] failed to sign %s: %s\n", file, err)
fails++
} else {
fmt.Printf("[SIGN] signed %s with %s\n", file, getSignedBySingle(fileData, trustStore))
signed++
}
default:
// File access error.
fmt.Printf("[FAIL] failed to access %s: %s\n", sigFile, err)
fails++
}
}
if verified > 0 {
fmt.Printf("[STAT] verified %d files", verified)
}
if signed > 0 {
fmt.Printf("[STAT] signed %d files", signed)
}
if fails > 0 {
return fmt.Errorf("signing failed on %d files", fails)
}
return nil
}
var (
trustStoreDir string
trustStoreKeyring string
)
func setupTrustStore() (trustStore truststores.ExtendedTrustStore, err error) {
// Get trust store directory.
if trustStoreDir == "" {
trustStoreDir, _ = os.LookupEnv("JESS_TS_DIR")
if trustStoreDir == "" {
trustStoreDir, _ = os.LookupEnv("JESS_TSDIR")
}
}
if trustStoreDir != "" {
trustStore, err = truststores.NewDirTrustStore(trustStoreDir)
if err != nil {
return nil, err
}
}
// Get trust store keyring.
if trustStore == nil {
if trustStoreKeyring == "" {
trustStoreKeyring, _ = os.LookupEnv("JESS_TS_KEYRING")
if trustStoreKeyring == "" {
trustStoreKeyring, _ = os.LookupEnv("JESS_TSKEYRING")
}
}
if trustStoreKeyring != "" {
trustStore, err = truststores.NewKeyringTrustStore(trustStoreKeyring)
if err != nil {
return nil, err
}
}
}
// Truststore is mandatory.
if trustStore == nil {
return nil, errors.New("no truststore configured, please pass arguments or use env variables")
}
return trustStore, nil
}
func getSignedByMany(fds []*filesig.FileData, trustStore jess.TrustStore) string {
signedBy := make([]string, 0, len(fds))
for _, fd := range fds {
if sig := fd.Signature(); sig != nil {
for _, seal := range sig.Signatures {
if signet, err := trustStore.GetSignet(seal.ID, true); err == nil {
signedBy = append(signedBy, fmt.Sprintf("%s (%s)", signet.Info.Name, seal.ID))
} else {
signedBy = append(signedBy, seal.ID)
}
}
}
}
return strings.Join(signedBy, " and ")
}
func getSignedBySingle(fd *filesig.FileData, trustStore jess.TrustStore) string {
if sig := fd.Signature(); sig != nil {
signedBy := make([]string, 0, len(sig.Signatures))
for _, seal := range sig.Signatures {
if signet, err := trustStore.GetSignet(seal.ID, true); err == nil {
signedBy = append(signedBy, fmt.Sprintf("%s (%s)", signet.Info.Name, seal.ID))
} else {
signedBy = append(signedBy, seal.ID)
}
}
return strings.Join(signedBy, " and ")
}
return ""
}

View file

@ -59,18 +59,25 @@
],
"desktopTemplate": "../../../packaging/linux/portmaster.desktop",
"files": {
// Service file
"/usr/lib/systemd/system/portmaster.service": "../../../packaging/linux/portmaster.service",
"/usr/lib/portmaster/bin-index.json": "binaries/bin-index.json",
"/usr/lib/portmaster/portmaster-core": "binaries/portmaster-core",
"/usr/lib/portmaster/portmaster.zip": "binaries/portmaster.zip",
"/usr/lib/portmaster/assets.zip": "binaries/assets.zip",
"/var/lib/portmaster/intel/intel-index.json": "binaries/intel-index.json",
"/var/lib/portmaster/intel/base.dsdl": "binaries/base.dsdl",
"/var/lib/portmaster/intel/geoipv4.mmdb": "binaries/geoipv4.mmdb",
"/var/lib/portmaster/intel/geoipv6.mmdb": "binaries/geoipv6.mmdb",
"/var/lib/portmaster/intel/index.dsd": "binaries/index.dsd",
"/var/lib/portmaster/intel/intermediate.dsdl": "binaries/intermediate.dsdl",
"/var/lib/portmaster/intel/urgent.dsdl": "binaries/urgent.dsdl",
// Binary files
"/usr/lib/portmaster/bin-index.json": "binary/bin-index.json",
"/usr/lib/portmaster/portmaster-core": "binary/portmaster-core",
"/usr/lib/portmaster/portmaster.zip": "binary/portmaster.zip",
"/usr/lib/portmaster/assets.zip": "binary/assets.zip",
// Intel files
"/var/lib/portmaster/intel/intel-index.json": "intel/intel-index.json",
"/var/lib/portmaster/intel/base.dsdl": "intel/base.dsdl",
"/var/lib/portmaster/intel/geoipv4.mmdb": "intel/geoipv4.mmdb",
"/var/lib/portmaster/intel/geoipv6.mmdb": "intel/geoipv6.mmdb",
"/var/lib/portmaster/intel/index.dsd": "intel/index.dsd",
"/var/lib/portmaster/intel/intermediate.dsdl": "intel/intermediate.dsdl",
"/var/lib/portmaster/intel/urgent.dsdl": "intel/urgent.dsdl",
// Shortcut
"/etc/xdg/autostart/portmaster.desktop": "../../../packaging/linux/portmaster-autostart.desktop"
},
"postInstallScript": "../../../packaging/linux/postinst",
@ -83,18 +90,25 @@
"desktopTemplate": "../../../packaging/linux/portmaster.desktop",
"release": "1",
"files": {
// Service file
"/usr/lib/systemd/system/portmaster.service": "../../../packaging/linux/portmaster.service",
"/usr/lib/portmaster/bin-index.json": "binaries/bin-index.json",
"/usr/lib/portmaster/portmaster-core": "binaries/portmaster-core",
"/usr/lib/portmaster/portmaster.zip": "binaries/portmaster.zip",
"/usr/lib/portmaster/assets.zip": "binaries/assets.zip",
"/var/lib/portmaster/intel/intel-index.json": "binaries/intel-index.json",
"/var/lib/portmaster/intel/base.dsdl": "binaries/base.dsdl",
"/var/lib/portmaster/intel/geoipv4.mmdb": "binaries/geoipv4.mmdb",
"/var/lib/portmaster/intel/geoipv6.mmdb": "binaries/geoipv6.mmdb",
"/var/lib/portmaster/intel/index.dsd": "binaries/index.dsd",
"/var/lib/portmaster/intel/intermediate.dsdl": "binaries/intermediate.dsdl",
"/var/lib/portmaster/intel/urgent.dsdl": "binaries/urgent.dsdl",
// Binary files
"/usr/lib/portmaster/bin-index.json": "binary/bin-index.json",
"/usr/lib/portmaster/portmaster-core": "binary/portmaster-core",
"/usr/lib/portmaster/portmaster.zip": "binary/portmaster.zip",
"/usr/lib/portmaster/assets.zip": "binary/assets.zip",
// Intel files
"/var/lib/portmaster/intel/intel-index.json": "intel/intel-index.json",
"/var/lib/portmaster/intel/base.dsdl": "intel/base.dsdl",
"/var/lib/portmaster/intel/geoipv4.mmdb": "intel/geoipv4.mmdb",
"/var/lib/portmaster/intel/geoipv6.mmdb": "intel/geoipv6.mmdb",
"/var/lib/portmaster/intel/index.dsd": "intel/index.dsd",
"/var/lib/portmaster/intel/intermediate.dsdl": "intel/intermediate.dsdl",
"/var/lib/portmaster/intel/urgent.dsdl": "intel/urgent.dsdl",
// Shortcut
"/etc/xdg/autostart/portmaster.desktop": "../../../packaging/linux/portmaster-autostart.desktop"
},
"postInstallScript": "../../../packaging/linux/postinst",

View file

@ -120,3 +120,57 @@ func checkIfFileIsValid(filename string, artifact Artifact) (bool, error) {
}
return true, nil
}
// GenerateBundleFromDir generates a bundle from a given folder.
func GenerateBundleFromDir(name string, version string, properties map[string]Artifact, dirPath string) (*Bundle, error) {
files, err := os.ReadDir(dirPath)
if err != nil {
return nil, err
}
artifacts := make([]Artifact, 0, len(files))
for _, f := range files {
// Skip dirs
if f.IsDir() {
continue
}
artifact := Artifact{
Filename: f.Name(),
}
predefined, ok := properties[f.Name()]
// Check if caller supplied predefined settings for this artifact.
if ok {
// File that have compression may have different filename and artifact filename. (because of the extension)
// If caller did not specify the artifact filename set it as the same as the filename.
if predefined.Filename == "" {
predefined.Filename = f.Name()
}
artifact = predefined
}
content, err := os.ReadFile(filepath.Join(dirPath, f.Name()))
if err != nil {
return nil, err
}
// Decompress if compression was applied to the file.
if artifact.Unpack != "" {
content, err = unpack(artifact.Unpack, content)
if err != nil {
return nil, err
}
}
hash := sha256.Sum256(content)
hashStr := hex.EncodeToString(hash[:])
artifact.SHA256 = hashStr
artifacts = append(artifacts, artifact)
}
return &Bundle{
Name: name,
Version: version,
Artifacts: artifacts,
Published: time.Now(),
}, nil
}