Update Earthfile to hopefully get cross-compilation for rust to work

Earthfile

@@ -8,6 +8,7 @@ ARG --global outputDir = "./dist"
 # The list of rust targets we support. They will be automatically converted
 # to GOOS, GOARCH and GOARM when building go binaries. See the +RUST_TO_GO_ARCH_STRING
 # helper method at the bottom of the file.
+
 ARG --global architectures = "x86_64-unknown-linux-gnu" \
                              "aarch64-unknown-linux-gnu" \
                              "armv7-unknown-linux-gnueabihf" \
@@ -22,6 +23,9 @@ go-deps:
     FROM golang:${go_version}-${distro}
     WORKDIR /go-workdir
 
+    # We need the git cli to extract version information for go-builds
+    RUN apk add git
+
     # These cache dirs will be used in later test and build targets
     # to persist cached go packages.
     #
@@ -54,6 +58,18 @@ go-base:
     # ./assets
     COPY assets ./assets
 
+    # Copy the git folder and extract version information
+    COPY .git ./.git
+
+    LET version = $(git tag --points-at)
+    IF [ "${version}" = "" ]
+        SET version = $(git describe --tags --abbrev=0)
+    END
+    ENV VERSION="${version}"
+
+    LET source = $(git remote -v | cut -f2 | cut -d" " -f1 | head -n 1)
+    ENV SOURCE="${source}"
+
 # updates all go dependencies and runs go mod tidy, saving go.mod and go.sum locally.
 update-go-deps:
     FROM +go-base
@@ -82,9 +98,6 @@ build-go:
     ARG GOARM
     ARG CMDS=portmaster-start portmaster-core hub notifier
 
-    # Get the current version
-    DO +GET_VERSION
-
     CACHE --sharing shared "$GOCACHE"
     CACHE --sharing shared "$GOMODCACHE"
 
@@ -97,12 +110,12 @@ build-go:
 
     # Build all go binaries from the specified in CMDS
     FOR bin IN $CMDS
-        RUN go build  -o "/tmp/build/" ./cmds/${bin}
+        RUN go build  -ldflags="-X github.com/safing/portbase/info.version=${VERSION} -X github.com/safing/portbase/info.buildSource=${SOURCE}" -o "/tmp/build/" ./cmds/${bin}
     END
 
-    FOR bin IN $(ls -1 "/tmp/build/")
+    DO +GO_ARCH_STRING --goos="${GOOS}" --goarch="${GOARCH}" --goarm="${GOARM}"
-        DO +GO_ARCH_STRING --goos="${GOOS}" --goarch="${GOARCH}" --goarm="${GOARM}"
 
+    FOR bin IN $(ls -1 "/tmp/build/")
         SAVE ARTIFACT "/tmp/build/${bin}" AS LOCAL "${outputDir}/${GO_ARCH_STRING}/${bin}"
     END
 
@@ -129,7 +142,8 @@ test-go:
     END
 
 test-go-all-platforms:
-    LOCALLY 
+    FROM alpine:3.18
+
     FOR arch IN ${architectures}
         DO +RUST_TO_GO_ARCH_STRING --rustTarget="${arch}"
         BUILD +test-go --GOARCH="${GOARCH}" --GOOS="${GOOS}" --GOARM="${GOARM}"
@@ -137,9 +151,19 @@ test-go-all-platforms:
 
 # Builds portmaster-start, portmaster-core, hub and notifier for all supported platforms
 build-go-release:
-    LOCALLY
+    FROM alpine:3.18
+
     FOR arch IN ${architectures}
         DO +RUST_TO_GO_ARCH_STRING --rustTarget="${arch}"
+
+        IF [ -z GOARCH ]
+            RUN echo "Failed to extract GOARCH for ${arch}"; exit 1
+        END
+
+        IF [ -z GOOS ]
+            RUN echo "Failed to extract GOOS for ${arch}"; exit 1
+        END
+
         BUILD +build-go --GOARCH="${GOARCH}" --GOOS="${GOOS}" --GOARM="${GOARM}"
     END
 
@@ -207,7 +231,12 @@ angular-dev:
 rust-base:
     FROM rust:1.76-bookworm
 
+    RUN dpkg --add-architecture armhf
+    RUN dpkg --add-architecture arm64
+
     RUN apt-get update -qq
+
+    # Tools and libraries required for cross-compilation
     RUN apt-get install --no-install-recommends -qq \
         autoconf \
         autotools-dev \
@@ -215,28 +244,66 @@ rust-base:
         clang \
         cmake \
         bsdmainutils \
+        gcc-multilib \
+        linux-libc-dev \
+        linux-libc-dev-amd64-cross \
+        linux-libc-dev-arm64-cross \
+        linux-libc-dev-armel-cross \
+        linux-libc-dev-armhf-cross \
+        build-essential \
+        curl \
+        wget \
+        file \
+        libsoup-3.0-dev \
+        libwebkit2gtk-4.1-dev 
+
+    # Install library dependencies for all supported architectures
+    # required for succesfully linking.
+    FOR arch IN amd64 arm64 armhf
+        RUN apt-get install --no-install-recommends -qq \
+            libsoup-3.0-0:${arch} \
+            libwebkit2gtk-4.1-0:${arch} \
+            libssl3:${arch} \
+            libayatana-appindicator3-1:${arch} \
+            librsvg2-bin:${arch} \
+            libgtk-3-0:${arch} \
+            libjavascriptcoregtk-4.1-0:${arch}  \
+            libssl-dev:${arch} \
+            libayatana-appindicator3-dev:${arch} \
+            librsvg2-dev:${arch} \
+            libgtk-3-dev:${arch} \
+            libjavascriptcoregtk-4.1-dev:${arch}  
+   END
+
+   # Note(ppacher): I've no idea why we need to explicitly create those symlinks:
+   # Some how all the other libs work but libsoup and libwebkit2gtk do not create the link file
+   RUN cd /usr/lib/aarch64-linux-gnu && \
+        ln -s libwebkit2gtk-4.1.so.0 libwebkit2gtk-4.1.so && \
+        ln -s libsoup-3.0.so.0 libsoup-3.0.so 
+
+   RUN cd /usr/lib/arm-linux-gnueabihf && \
+        ln -s libwebkit2gtk-4.1.so.0 libwebkit2gtk-4.1.so && \
+        ln -s libsoup-3.0.so.0 libsoup-3.0.so 
+
+    # For what ever reason trying to install the gcc compilers together with the above
+    # command makes apt fail due to conflicts with gcc-multilib. Installing in a separate
+    # step seems to work ...
+    RUN apt-get install --no-install-recommends -qq \
         g++-mingw-w64-x86-64 \
         gcc-aarch64-linux-gnu \
         gcc-arm-none-eabi \
         gcc-arm-linux-gnueabi \
         gcc-arm-linux-gnueabihf \
-        libgtk-3-dev \
+        libc6-dev-arm64-cross \
-        libjavascriptcoregtk-4.1-dev \
+        libc6-dev-armel-cross \
-        libsoup-3.0-dev \
+        libc6-dev-armhf-cross \
-        libwebkit2gtk-4.1-dev \
+        libc6-dev-amd64-cross
-        build-essential \
-        curl \
-        wget \
-        file \
-        libssl-dev \
-        libayatana-appindicator3-dev \
-        librsvg2-dev
 
     # Add some required rustup components
     RUN rustup component add clippy
     RUN rustup component add rustfmt
 
-    # Install toolchains and targets
+    # Install architecture targets
     FOR arch IN ${architectures}
         RUN rustup target add ${arch}
     END
@@ -246,6 +313,9 @@ rust-base:
     # For now we need tauri-cli 1.5 for bulding
     DO rust+CARGO --args="install tauri-cli --version ^1.5.11"
 
+    # Required for cross compilation to work.
+    ENV PKG_CONFIG_ALLOW_CROSS=1
+
 tauri-src:
     FROM +rust-base
 
@@ -268,28 +338,34 @@ build-tauri:
     # we need to do some magic here because tauri expects the binaries to include the rust target tripple.
     # We already knwo that triple because it's a required argument. From that triple, we use +RUST_TO_GO_ARCH_STRING
     # function from below to parse the triple and guess wich GOOS and GOARCH we need.
-    IF [ "${bundle}" != "none" ] 
+    RUN mkdir /tmp/gobuild
-        RUN mkdir /tmp/gobuild
+    RUN mkdir ./binaries
-        RUN mkdir ./binaries
 
-        DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
+    DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
-        RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
+    RUN echo "GOOS=${GOOS} GOARCH=${GOARCH} GOARM=${GOARM} GO_ARCH_STRING=${GO_ARCH_STRING}"
 
-        COPY (+build-go/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild
+    # Our tauri app has externalBins configured so tauri will try to embed them when it finished compiling
+    # the app. Make sure we copy portmaster-start and portmaster-core in all architectures supported.
+    # See documentation for externalBins for more information on how tauri searches for the binaries.
 
-        LET dest=""
+    COPY (+build-go/output --GOOS="${GOOS}" --CMDS="portmaster-start portmaster-core" --GOARCH="${GOARCH}" --GOARM="${GOARM}") /tmp/gobuild
-        FOR bin IN $(ls /tmp/gobuild)
-            SET dest="./binaries/${bin}-${target}"
 
-            IF [ -z "${bin##*.exe}" ]
+    # Place them in the correct folder with the rust target tripple attached.
-                SET dest = "./binaries/${bin%.*}-${target}.exe"
+    LET dest=""
-            END
+    FOR bin IN $(ls /tmp/gobuild)
+        SET dest="./binaries/${bin}-${target}"
 
-            RUN echo "Copying ${bin} to ${dest}"
+        IF [ -z "${bin##*.exe}" ]
-            RUN cp "/tmp/gobuild/${bin}" "${dest}"
+            SET dest = "./binaries/${bin%.*}-${target}.exe"
         END
+
+        RUN echo "Copying ${bin} to ${dest}"
+        RUN cp "/tmp/gobuild/${bin}" "${dest}"
     END
 
+    # Just for debugging ...
+    RUN ls -R ./binaries
+
     # The following is exected to work but doesn't. for whatever reason cargo-sweep errors out on the windows-toolchain.
     #
     #   DO rust+CARGO --args="tauri build --bundles none --ci --target=${target}" --output="release/[^/\.]+"
@@ -300,10 +376,23 @@ build-tauri:
     RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo tauri build --bundles "${bundle}" --ci --target="${target}"
     DO rust+COPY_OUTPUT --output="${output}"
 
+    # BUG(cross-compilation):
+    #
+    # The above command seems to correctly compile for all architectures we want to support but fails during
+    # linking since the target libaries are not available for the requested platforms. Maybe we need to download
+    # the, manually ...
+    #
+    # The earthly rust lib also has support for using cross-rs for cross-compilation but that fails due to the
+    # fact that cross-rs base docker images used for building are heavily outdated (latest = ubunut:16.0, main = ubuntu:20.04)
+    # which does not ship recent enough glib versions (our glib dependency needs glib>2.70 but ubunut:20.04 only ships 2.64)
+    #
+    # The following would use the CROSS function from the earthly lib, this 
+    # DO rust+CROSS --target="${target}"
+
     RUN ls target
 
 tauri-release:
-    LOCALLY
+    FROM alpine:3.18
 
     ARG bundle="none"
 
@@ -311,9 +400,18 @@ tauri-release:
         BUILD +build-tauri --target="${arch}" --bundle="${bundle}"
     END
 
-release:
+build-all:
     BUILD +build-go-release
-    BUILD +angular-release
+    BUILD +tauri-release
+
+release:
+    LOCALLY
+
+    IF ! git diff --quiet
+        RUN echo -e "\033[1;31m Refusing to release a dirty git repository. Please commit your local changes first! \033[0m" ; exit 1
+    END
+
+    BUILD +build-all
 
 
 # Takes GOOS, GOARCH and optionally GOARM and creates a string representation for file-names.
@@ -375,22 +473,3 @@ RUST_TO_GO_ARCH_STRING:
     ENV GOARM="${goarm}"
 
     DO +GO_ARCH_STRING --goos="${goos}" --goarch="${goarch}" --goarm="${goarm}"
-
-GET_VERSION:
-    FUNCTION
-    LOCALLY
-
-    LET VERSION=$(git tag --points-at)
-    IF [ -z "${VERSION}"]
-        SET VERSION=$(git describe --tags --abbrev=0)§dev§build
-    ELSE IF ! git diff --quite
-        SET VERSION="${VERSION}§dev§build"
-    END
-
-    RUN echo "Version is ${VERSION}"
-    ENV VERSION="${VERSION}"
-
-test:
-    LOCALLY
-
-    DO +GET_VERSION

cmds/portmaster-core/build

@@ -1,60 +0,0 @@
-#!/bin/bash
-
-# get build data
-if [[ "$BUILD_COMMIT" == "" ]]; then
-  BUILD_COMMIT=$(git describe --all --long --abbrev=99 --dirty 2>/dev/null)
-fi
-if [[ "$BUILD_USER" == "" ]]; then
-  BUILD_USER=$(id -un)
-fi
-if [[ "$BUILD_HOST" == "" ]]; then
-  BUILD_HOST=$(hostname -f)
-fi
-if [[ "$BUILD_DATE" == "" ]]; then
-  BUILD_DATE=$(date +%d.%m.%Y)
-fi
-if [[ "$BUILD_SOURCE" == "" ]]; then
-  BUILD_SOURCE=$(git remote -v | grep origin | cut -f2 | cut -d" " -f1 | head -n 1)
-fi
-if [[ "$BUILD_SOURCE" == "" ]]; then
-  BUILD_SOURCE=$(git remote -v | cut -f2 | cut -d" " -f1 | head -n 1)
-fi
-BUILD_BUILDOPTIONS=$(echo $* | sed "s/ /§/g")
-
-# check
-if [[ "$BUILD_COMMIT" == "" ]]; then
-  echo "could not automatically determine BUILD_COMMIT, please supply manually as environment variable."
-  exit 1
-fi
-if [[ "$BUILD_USER" == "" ]]; then
-  echo "could not automatically determine BUILD_USER, please supply manually as environment variable."
-  exit 1
-fi
-if [[ "$BUILD_HOST" == "" ]]; then
-  echo "could not automatically determine BUILD_HOST, please supply manually as environment variable."
-  exit 1
-fi
-if [[ "$BUILD_DATE" == "" ]]; then
-  echo "could not automatically determine BUILD_DATE, please supply manually as environment variable."
-  exit 1
-fi
-if [[ "$BUILD_SOURCE" == "" ]]; then
-  echo "could not automatically determine BUILD_SOURCE, please supply manually as environment variable."
-  exit 1
-fi
-
-echo "Please notice, that this build script includes metadata into the build."
-echo "This information is useful for debugging and license compliance."
-echo "Run the compiled binary with the -version flag to see the information included."
-
-if [[ $1 == "dev" ]]; then
-  shift
-  export CGO_ENABLED=1
-  DEV="-race"
-else
-  export CGO_ENABLED=0
-fi
-
-# build
-BUILD_PATH="github.com/safing/portbase/info"
-go build $DEV -ldflags "-X ${BUILD_PATH}.commit=${BUILD_COMMIT} -X ${BUILD_PATH}.buildOptions=${BUILD_BUILDOPTIONS} -X ${BUILD_PATH}.buildUser=${BUILD_USER} -X ${BUILD_PATH}.buildHost=${BUILD_HOST} -X ${BUILD_PATH}.buildDate=${BUILD_DATE} -X ${BUILD_PATH}.buildSource=${BUILD_SOURCE}" "$@"

cmds/portmaster-core/pack

@@ -1,123 +0,0 @@
-#!/bin/bash
-
-baseDir="$( cd "$(dirname "$0")" && pwd )"
-cd "$baseDir"
-
-COL_OFF="\033[0m"
-COL_BOLD="\033[01;01m"
-COL_RED="\033[31m"
-COL_GREEN="\033[32m"
-COL_YELLOW="\033[33m"
-
-destDirPart1="../../dist"
-destDirPart2="core"
-
-function prep {
-  # output
-  output="portmaster-core"
-  # get version
-  version=$(grep "info.Set" main.go | cut -d'"' -f4)
-  # build versioned file name
-  filename="portmaster-core_v${version//./-}"
-  # platform
-  platform="${GOOS}_${GOARCH}"
-  if [[ $GOOS == "windows" ]]; then
-    filename="${filename}.exe"
-    output="${output}.exe"
-  fi
-  # build destination path
-  destPath=${destDirPart1}/${platform}/${destDirPart2}/$filename
-}
-
-function check {
-  prep
-
-  # check if file exists
-  if [[ -f $destPath ]]; then
-    echo "[core] $platform v$version already built"
-  else
-    echo -e "${COL_BOLD}[core] $platform v$version${COL_OFF}"
-  fi
-}
-
-function build {
-  prep
-
-  # check if file exists
-  if [[ -f $destPath ]]; then
-    echo "[core] $platform already built in v$version, skipping..."
-    return
-  fi
-
-  # build
-  ./build
-  if [[ $? -ne 0 ]]; then
-    echo -e "\n${COL_BOLD}[core] $platform v$version: ${COL_RED}BUILD FAILED.${COL_OFF}"
-    exit 1
-  fi
-  mkdir -p $(dirname $destPath)
-  cp $output $destPath
-  echo -e "\n${COL_BOLD}[core] $platform v$version: ${COL_GREEN}successfully built.${COL_OFF}"
-}
-
-function reset {
-  prep
-
-  # delete if file exists
-  if [[ -f $destPath ]]; then
-    rm $destPath
-    echo "[core] $platform v$version deleted."
-  fi
-}
-
-function check_all {
-  GOOS=linux GOARCH=amd64 check
-  GOOS=windows GOARCH=amd64 check
-  GOOS=darwin GOARCH=amd64 check
-  GOOS=linux GOARCH=arm64 check
-  GOOS=windows GOARCH=arm64 check
-  GOOS=darwin GOARCH=arm64 check
-}
-
-function build_all {
-  GOOS=linux GOARCH=amd64 build
-  GOOS=windows GOARCH=amd64 build
-  GOOS=darwin GOARCH=amd64 build
-  GOOS=linux GOARCH=arm64 build
-  GOOS=windows GOARCH=arm64 build
-  GOOS=darwin GOARCH=arm64 build
-}
-
-function reset_all {
-  GOOS=linux GOARCH=amd64 reset
-  GOOS=windows GOARCH=amd64 reset
-  GOOS=darwin GOARCH=amd64 reset
-  GOOS=linux GOARCH=arm64 reset
-  GOOS=windows GOARCH=arm64 reset
-  GOOS=darwin GOARCH=arm64 reset
-}
-
-case $1 in
-  "check" )
-    check_all
-    ;;
-  "build" )
-    build_all
-    ;;
-  "reset" )
-    reset_all
-    ;;
-  * )
-    echo ""
-    echo "build list:"
-    echo ""
-    check_all
-    echo ""
-    read -p "press [Enter] to start building" x
-    echo ""
-    build_all
-    echo ""
-    echo "finished building."
-    echo ""
-    ;;
-esac

desktop/tauri/.cargo/config.toml

@@ -0,0 +1,7 @@
+[target.aarch64-unknown-linux-gnu]
+linker = "aarch64-linux-gnu-gcc"
+rustflags = ["-C", "link-args=-L/usr/lib/aarch64-linux-gnu/"]
+
+[target.armv7-unknown-linux-gnueabihf]
+linker = "arm-linux-gnueabihf-gcc"
+rustflags = ["-C", "link-args=-L/usr/lib/arm-linux-gnueabihf/"]

desktop/tauri/src-tauri/Cross.toml

@@ -0,0 +1,7 @@
+[target.aarch64-unknown-linux-gnu]
+image = "ghcr.io/cross-rs/aarch64-unknown-linux-gnu:main"
+
+pre-build = [
+    "dpkg --add-architecture $CROSS_DEB_ARCH", 
+    "apt-get update && apt-get --assume-yes install libssl-dev:$CROSS_DEB_ARCH libjavascriptcoregtk-4.0-dev:$CROSS_DEB_ARCH librsvg2-dev libayatana-appindicator3-dev libwebkit2gtk-4.0-dev libsoup2.4-dev libgtk-3-dev"
+]

desktop/tauri/src-tauri/src/xdg/mod.rs

@@ -404,7 +404,7 @@ fn get_icon_as_png_dataurl(name: &str, size: i8) -> Result<(String, String)> {
 
             let icon_info = gtk_icon_theme_lookup_icon(
                 GTK_DEFAULT_THEME.unwrap(),
-                c_str.as_ptr() as *const i8,
+                c_str.as_ptr() as *const u8,
                 size as c_int,
                 0,
             );

desktop/tauri/src-tauri/tauri.conf.json

@@ -47,8 +47,8 @@
         "depends": []
       },
       "externalBin": [
-        "binaries/portmaster-start",
+        "../binaries/portmaster-start",
-        "binaries/portmaster-core"
+        "../binaries/portmaster-core"
       ],
       "icon": [
         "../assets/icons/pm_dark_512.png",