[WIP] Improve CI build

This commit is contained in:
Vladimir Stoilov 2024-10-07 11:26:27 +03:00
parent 8e1f3c0ed9
commit 71f67a8936
No known key found for this signature in database
GPG key ID: 2F190B67A43A81AF
8 changed files with 327 additions and 104 deletions

View file

@ -3,7 +3,7 @@ VERSION --arg-scope-and-set --global-cache 0.8
ARG --global go_version = 1.22
ARG --global node_version = 18
ARG --global rust_version = 1.79
ARG --global tauri_version = "2.0.0-rc.8"
ARG --global tauri_version = "2.0.1"
ARG --global golangci_lint_version = 1.57.1
ARG --global go_builder_image = "golang:${go_version}-alpine"
@ -509,7 +509,65 @@ tauri-lint:
WORKDIR /app/desktop/tauri/src-tauri
RUN --mount=$EARTHLY_RUST_TARGET_CACHE cargo clippy --all-targets --all-features -- -D warnings
tauri-bundle-linux:
release-prep:
FROM +rust-base
# Linux specific
COPY (+tauri-build/output/portmaster --target="x86_64-unknown-linux-gnu") ./output/binary/linux_amd64/portmaster
COPY (+go-build/output/portmaster-core --GOARCH=amd64 --GOOS=linux --CMDS=portmaster-core) ./output/binary/linux_amd64/portmaster-core
# Windows specific
COPY (+tauri-build/output/portmaster.exe --target="x86_64-pc-windows-gnu") ./output/binary/windows_amd64/portmaster.exe
COPY (+go-build/output/portmaster-core.exe --GOARCH=amd64 --GOOS=windows --CMDS=portmaster-core) ./output/binary/windows_amd64/portmaster-core.exe
# TODO(vladimir): figure out a way to get the lastest release of the kext.
RUN touch ./output/binary/windows_amd64/portmaster-kext.sys
# All platforms
COPY (+assets/assets.zip) ./output/binary/all/assets.zip
COPY (+angular-project/output/portmaster.zip --project=portmaster --dist=./dist --configuration=production --baseHref=/ui/modules/portmaster/) ./output/binary/all/portmaster.zip
# Intel
# TODO(vladimir): figure out a way to download all latest intel data.
RUN mkdir -p ./output/intel
RUN wget -O ./output/intel/geoipv4.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv4_v20240529-0-1.mmdb.gz" && \
wget -O ./output/intel/geoipv6.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv6_v20240529-0-1.mmdb.gz"
RUN touch "./output/intel/index.dsd"
RUN touch "./output/intel/base.dsdl"
RUN touch "./output/intel/intermediate.dsdl"
RUN touch "./output/intel/urgent.dsdl"
COPY (+go-build/output/updatemgr --GOARCH=amd64 --GOOS=linux --CMDS=updatemgr) ./updatemgr
RUN ./updatemgr -dir "./output/binary" -name "Binary" > ./output/binary/bin-index.json
RUN ./updatemgr -dir "./output/intel" -name "Intel" > ./output/intel/intel-index.json
# Intel Extracted (needed for the installers)
RUN mkdir -p ./output/intel_decompressed
RUN cp ./output/intel/intel-index.json ./output/intel_decompressed/intel-index.json
RUN gzip -dc ./output/intel/geoipv4.mmdb.gz > ./output/intel_decompressed/geoipv4.mmdb
RUN gzip -dc ./output/intel/geoipv6.mmdb.gz > ./output/intel_decompressed/geoipv6.mmdb
RUN touch "./output/intel_decompressed/index.dsd"
RUN touch "./output/intel_decompressed/base.dsdl"
RUN touch "./output/intel_decompressed/intermediate.dsdl"
RUN touch "./output/intel_decompressed/urgent.dsdl"
# Save all artifacts to output folder
SAVE ARTIFACT --if-exists --keep-ts "output/binary/bin-index.json" AS LOCAL "${outputDir}/binary/bin-index.json"
SAVE ARTIFACT --if-exists --keep-ts "output/binary/all/*" AS LOCAL "${outputDir}/binary/all/"
SAVE ARTIFACT --if-exists --keep-ts "output/binary/linux_amd64/*" AS LOCAL "${outputDir}/binary/linux_amd64/"
SAVE ARTIFACT --if-exists --keep-ts "output/binary/windows_amd64/*" AS LOCAL "${outputDir}/binary/windows_amd64/"
SAVE ARTIFACT --if-exists --keep-ts "output/intel/*" AS LOCAL "${outputDir}/intel/"
SAVE ARTIFACT --if-exists --keep-ts "output/intel_decompressed/*" AS LOCAL "${outputDir}/intel_decompressed/"
# Save all artifacts to the container output folder so other containers can access it.
SAVE ARTIFACT --if-exists --keep-ts "output/binary/bin-index.json" "output/binary/bin-index.json"
SAVE ARTIFACT --if-exists --keep-ts "output/binary/all/*" "output/binary/all/"
SAVE ARTIFACT --if-exists --keep-ts "output/binary/linux_amd64/*" "output/binary/linux_amd64/"
SAVE ARTIFACT --if-exists --keep-ts "output/binary/windows_amd64/*" "output/binary/windows_amd64/"
SAVE ARTIFACT --if-exists --keep-ts "output/intel/*" "output/intel/"
SAVE ARTIFACT --if-exists --keep-ts "output/intel_decompressed/*" "output/intel_decompressed/"
installer-linux:
FROM +rust-base
# ARG --required target
ARG target="x86_64-unknown-linux-gnu"
@ -523,42 +581,21 @@ tauri-bundle-linux:
SAVE IMAGE --cache-hint
DO +RUST_TO_GO_ARCH_STRING --rustTarget="${target}"
# Build and copy the binaries
RUN mkdir -p target/${target}/release
COPY (+tauri-build/output/portmaster --target=x86_64-unknown-linux-gnu) ./target/${target}/release/portmaster
COPY (+release-prep/output/binary/linux_amd64/portmaster) ./target/${target}/release/portmaster
RUN mkdir -p binary
COPY (+go-build/output/portmaster-core --GOARCH=amd64 --GOOS=linux --CMDS=portmaster-core) ./binary/portmaster-core
COPY (+assets/assets.zip) ./binary/assets.zip
COPY (+angular-project/output/portmaster.zip --project=portmaster --dist=./dist --configuration=production --baseHref=/ui/modules/portmaster/) ./binary/portmaster.zip
COPY (+release-prep/output/binary/bin-index.json) ./binary/bin-index.json
COPY (+release-prep/output/binary/linux_amd64/portmaster-core) ./binary/portmaster-core
COPY (+release-prep/output/binary/all/portmaster.zip) ./binary/portmaster.zip
COPY (+release-prep/output/binary/all/assets.zip) ./binary/assets.zip
# Download the intel data
RUN mkdir -p intel
RUN wget -O ./intel/geoipv4.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv4_v20240529-0-1.mmdb.gz" && \
wget -O ./intel/geoipv6.mmdb.gz "https://updates.safing.io/all/intel/geoip/geoipv6_v20240529-0-1.mmdb.gz" && \
gzip -d ./intel/geoipv4.mmdb.gz && \
gzip -d ./intel/geoipv6.mmdb.gz
RUN touch "./intel/index.dsd"
RUN touch "./intel/base.dsdl"
RUN touch "./intel/intermediate.dsdl"
RUN touch "./intel/urgent.dsdl"
# Generate index files
COPY (+go-build/output/updatemgr --GOARCH=amd64 --GOOS=linux --CMDS=updatemgr) ./updatemgr
RUN ./updatemgr -dir "./binary" -name "Binary" > ./binary/bin-index.json
RUN ./updatemgr -dir "./intel" -name "Intel" > ./intel/intel-index.json
RUN cat ./binary/bin-index.json
RUN cat ./intel/intel-index.json
COPY (+release-prep/output/intel_decompressed/*) ./intel/
# build the installers
RUN cargo tauri bundle --ci --target="${target}"

View file

@ -10,17 +10,21 @@ import (
)
var binaryMap = map[string]updates.Artifact{
"portmaster-core": {
Platform: "linux_amd64",
"geoipv4.mmdb.gz": {
Filename: "geoipv4.mmdb",
Unpack: "gz",
},
"portmaster-core.exe": {
Platform: "windows_amd64",
},
"portmaster-kext.sys": {
Platform: "windows_amd64",
"geoipv6.mmdb.gz": {
Filename: "geoipv6.mmdb",
Unpack: "gz",
},
}
var ignoreFiles = map[string]struct{}{
"bin-index.json": {},
"intel-index.json": {},
}
func main() {
dir := flag.String("dir", "", "path to the directory that contains the artifacts")
name := flag.String("name", "", "name of the bundle")
@ -36,7 +40,13 @@ func main() {
return
}
bundle, err := updates.GenerateBundleFromDir(*name, *version, binaryMap, *dir)
settings := updates.BundleFileSettings{
Name: *name,
Version: *version,
Properties: binaryMap,
IgnoreFiles: ignoreFiles,
}
bundle, err := updates.GenerateBundleFromDir(*dir, settings)
if err != nil {
fmt.Fprintf(os.Stderr, "failed to generate bundle: %s\n", err)
return

View file

@ -12,21 +12,21 @@ rust-version = "1.64"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[build-dependencies]
tauri-build = { version = "2.0.0-rc.7", features = [] }
tauri-build = { version = "2.0.1", features = [] }
[dependencies]
# Tauri
tauri = { version = "2.0.0-rc.8", features = ["tray-icon", "image-png", "config-json5", "devtools"] }
tauri-plugin-shell = "2.0.0-rc"
tauri-plugin-dialog = "2.0.0-rc"
tauri-plugin-clipboard-manager = "2.0.0-rc"
tauri-plugin-os = "2.0.0-rc"
tauri-plugin-single-instance = "2.0.0-rc"
tauri-plugin-notification = "2.0.0-rc"
tauri-plugin-log = "2.0.0-rc"
tauri-plugin-window-state = "2.0.0-rc"
tauri = { version = "2.0.1", features = ["tray-icon", "image-png", "config-json5", "devtools"] }
tauri-plugin-shell = "2.0.1"
tauri-plugin-dialog = "2.0.1"
tauri-plugin-clipboard-manager = "2.0.1"
tauri-plugin-os = "2.0.1"
tauri-plugin-single-instance = "2.0.1"
tauri-plugin-notification = "2.0.1"
tauri-plugin-log = "2.0.1"
tauri-plugin-window-state = "2.0.1"
tauri-cli = "2.0.0-rc.8"
tauri-cli = "2.0.1"
clap_lex = "0.7.2"
# General

View file

@ -139,7 +139,7 @@
{{/each~}}
</Component>
<Component Id="Path" Guid="{{path_component_guid}}" Win64="$(var.Win64)">
<File Id="Path" Source="{{app_exe_source}}" KeyPath="yes" Checksum="yes"/>
<File Id="Path" Source="{{main_binary_path}}" KeyPath="yes" Checksum="yes"/>
{{#each file_associations as |association| ~}}
{{#each association.ext as |ext| ~}}
<ProgId Id="{{../../product_name}}.{{ext}}" Advertise="yes" Description="{{association.description}}">

View file

@ -139,7 +139,7 @@
{{/each~}}
</Component>
<Component Id="Path" Guid="{{path_component_guid}}" Win64="$(var.Win64)">
<File Id="Path" Source="{{app_exe_source}}" KeyPath="yes" Checksum="yes"/>
<File Id="Path" Source="{{main_binary_path}}" KeyPath="yes" Checksum="yes"/>
{{#each file_associations as |association| ~}}
{{#each association.ext as |ext| ~}}
<ProgId Id="{{../../product_name}}.{{ext}}" Advertise="yes" Description="{{association.description}}">

View file

@ -0,0 +1,46 @@
# Save the current directory
$originalDirectory = Get-Location
$destinationDir = "desktop/tauri/src-tauri"
$binaryDir = "$destinationDir/binary"
$intelDir = "$destinationDir/intel"
# Make sure distination folder exists.
if (-not (Test-Path -Path $binaryDir)) {
New-Item -ItemType Directory -Path $binaryDir > $null
}
# Copy binary files
Copy-Item -Force -Path "dist/binary/bin-index.json" -Destination "$binaryDir/bin-index.json"
Copy-Item -Force -Path "dist/binary/windows_amd64/portmaster-core.exe" -Destination "$binaryDir/portmaster-core.exe"
Copy-Item -Force -Path "dist/binary/windows_amd64/portmaster-kext.sys" -Destination "$binaryDir/portmaster-kext.sys"
Copy-Item -Force -Path "dist/binary/all/portmaster.zip" -Destination "$binaryDir/portmaster.zip"
Copy-Item -Force -Path "dist/binary/all/assets.zip" -Destination "$binaryDir/assets.zip"
Copy-Item -Force -Path "dist/binary/windows_amd64/portmaster.exe" -Destination "$destinationDir/target/release/portmaster.exe"
# Make sure distination folder exists.
if (-not (Test-Path -Path $intelDir)) {
New-Item -ItemType Directory -Path $intelDir > $null
}
# Copy intel data
Copy-Item -Force -Path "dist/intel_decompressed/*" -Destination "$intelDir/"
Set-Location $destinationDir
# Download tauri-cli
Invoke-WebRequest -Uri https://github.com/tauri-apps/tauri/releases/download/tauri-cli-v2.0.1/cargo-tauri-x86_64-pc-windows-msvc.zip -OutFile tauri-cli.zip
Expand-Archive -Force tauri-cli.zip
./tauri-cli/cargo-tauri.exe bundle
$installerDist = "..\..\..\dist\windows_amd64\"
# Make sure distination folder exists.
if (-not (Test-Path -Path $installerDist)) {
New-Item -ItemType Directory -Path $installerDist > $null
}
Copy-Item -Path ".\target\release\bundle\msi\*" -Destination $installerDist
Copy-Item -Path ".\target\release\bundle\nsis\*" -Destination $installerDist
# Restore the original directory
Set-Location $originalDirectory

View file

@ -120,57 +120,3 @@ func checkIfFileIsValid(filename string, artifact Artifact) (bool, error) {
}
return true, nil
}
// GenerateBundleFromDir generates a bundle from a given folder.
func GenerateBundleFromDir(name string, version string, properties map[string]Artifact, dirPath string) (*Bundle, error) {
files, err := os.ReadDir(dirPath)
if err != nil {
return nil, err
}
artifacts := make([]Artifact, 0, len(files))
for _, f := range files {
// Skip dirs
if f.IsDir() {
continue
}
artifact := Artifact{
Filename: f.Name(),
}
predefined, ok := properties[f.Name()]
// Check if caller supplied predefined settings for this artifact.
if ok {
// File that have compression may have different filename and artifact filename. (because of the extension)
// If caller did not specify the artifact filename set it as the same as the filename.
if predefined.Filename == "" {
predefined.Filename = f.Name()
}
artifact = predefined
}
content, err := os.ReadFile(filepath.Join(dirPath, f.Name()))
if err != nil {
return nil, err
}
// Decompress if compression was applied to the file.
if artifact.Unpack != "" {
content, err = unpack(artifact.Unpack, content)
if err != nil {
return nil, err
}
}
hash := sha256.Sum256(content)
hashStr := hex.EncodeToString(hash[:])
artifact.SHA256 = hashStr
artifacts = append(artifacts, artifact)
}
return &Bundle{
Name: name,
Version: version,
Artifacts: artifacts,
Published: time.Now(),
}, nil
}

View file

@ -0,0 +1,184 @@
package updates
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"os"
"path"
"path/filepath"
"regexp"
"strings"
"time"
semver "github.com/hashicorp/go-version"
)
type BundleFileSettings struct {
Name string
Version string
Properties map[string]Artifact
IgnoreFiles map[string]struct{}
}
// GenerateBundleFromDir generates a bundle from a given folder.
func GenerateBundleFromDir(bundleDir string, settings BundleFileSettings) (*Bundle, error) {
bundleDirName := filepath.Base(bundleDir)
artifacts := make([]Artifact, 0, 5)
err := filepath.Walk(bundleDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
// Skip folders
if info.IsDir() {
return nil
}
identifier, version, ok := getIdentifierAndVersion(info.Name())
if !ok {
identifier = info.Name()
}
// Check if file is in the ignore list.
if _, ok := settings.IgnoreFiles[identifier]; ok {
return nil
}
artifact := Artifact{}
// Check if the caller provided properties for the artifact.
if p, ok := settings.Properties[identifier]; ok {
artifact = p
}
// Set filename of artifact if not set by the caller.
if artifact.Filename == "" {
artifact.Filename = identifier
}
artifact.Version = version
// Fill the platform of the artifact
parentDir := filepath.Base(filepath.Dir(path))
if parentDir != "all" && parentDir != bundleDirName {
artifact.Platform = parentDir
}
// Fill the hash
hash, err := getSHA256(path, artifact.Unpack)
if err != nil {
return fmt.Errorf("failed to calculate hash of file: %s %w", path, err)
}
artifact.SHA256 = hash
artifacts = append(artifacts, artifact)
return nil
})
if err != nil {
return nil, fmt.Errorf("failed to walk the dir: %w", err)
}
// Filter artifact so we have single version for each file
artifacts, err = selectLatestArtifacts(artifacts)
if err != nil {
return nil, fmt.Errorf("failed to select artifact version: %w", err)
}
return &Bundle{
Name: settings.Name,
Version: settings.Version,
Artifacts: artifacts,
Published: time.Now(),
}, nil
}
func selectLatestArtifacts(artifacts []Artifact) ([]Artifact, error) {
artifactsMap := make(map[string]Artifact)
for _, a := range artifacts {
// Make the key platform specific since there can be same filename for multiple platforms.
key := a.Filename + a.Platform
aMap, ok := artifactsMap[key]
if !ok {
artifactsMap[key] = a
continue
}
if aMap.Version == "" || a.Version == "" {
return nil, fmt.Errorf("invalid mix version and non versioned files for: %s", a.Filename)
}
mapVersion, err := semver.NewVersion(aMap.Version)
if err != nil {
return nil, fmt.Errorf("invalid version for artifact: %s", aMap.Filename)
}
artifactVersion, err := semver.NewVersion(a.Version)
if err != nil {
return nil, fmt.Errorf("invalid version for artifact: %s", a.Filename)
}
if mapVersion.LessThan(artifactVersion) {
artifactsMap[key] = a
}
}
artifactsFiltered := make([]Artifact, 0, len(artifactsMap))
for _, a := range artifactsMap {
artifactsFiltered = append(artifactsFiltered, a)
}
return artifactsFiltered, nil
}
func getSHA256(path string, unpackType string) (string, error) {
content, err := os.ReadFile(path)
if err != nil {
return "", err
}
// Decompress if compression was applied to the file.
if unpackType != "" {
content, err = unpack(unpackType, content)
if err != nil {
return "", err
}
}
// Calculate hash
hash := sha256.Sum256(content)
return hex.EncodeToString(hash[:]), nil
}
var (
fileVersionRegex = regexp.MustCompile(`_v[0-9]+-[0-9]+-[0-9]+(-[a-z]+)?`)
rawVersionRegex = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+(-[a-z]+)?$`)
)
func getIdentifierAndVersion(versionedPath string) (identifier, version string, ok bool) {
dirPath, filename := path.Split(versionedPath)
// Extract version from filename.
rawVersion := fileVersionRegex.FindString(filename)
if rawVersion == "" {
// No version present in file, making it invalid.
return "", "", false
}
// Trim the `_v` that gets caught by the regex and
// replace `-` with `.` to get the version string.
version = strings.Replace(strings.TrimLeft(rawVersion, "_v"), "-", ".", 2)
// Put the filename back together without version.
i := strings.Index(filename, rawVersion)
if i < 0 {
// extracted version not in string (impossible)
return "", "", false
}
filename = filename[:i] + filename[i+len(rawVersion):]
// Put the full path back together and return it.
// `dirPath + filename` is guaranteed by path.Split()
return dirPath + filename, version, true
}