Improve internal tooling

cmds/trafficgen/main.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"time"
+
+	"github.com/miekg/dns"
+
+	"github.com/safing/portbase/log"
+)
+
+const (
+	dnsResolver = "1.1.1.1:53"
+)
+
+var (
+	url      string
+	lookup   string
+	n        int
+	waitMsec int
+)
+
+func init() {
+	flag.StringVar(&url, "url", "", "send HTTP HEAD requests to this url")
+	flag.StringVar(&lookup, "lookup", "", fmt.Sprintf("query %s for this domains", dnsResolver))
+	flag.IntVar(&n, "n", 1000, "how many requests to make")
+	flag.IntVar(&waitMsec, "w", 100, "how many ms to wait between requests")
+}
+
+func main() {
+	// Parse flags
+	flag.Parse()
+	if url == "" && lookup == "" {
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	// Start logging.
+	err := log.Start()
+	if err != nil {
+		fmt.Printf("failed to start logging: %s\n", err)
+		os.Exit(1)
+	}
+	defer log.Shutdown()
+	log.SetLogLevel(log.TraceLevel)
+	log.Info("starting traffic generator")
+
+	// Execute requests
+	waitDuration := time.Duration(waitMsec) * time.Millisecond
+	for i := 0; i < n; i++ {
+		makeHTTPRequest()
+		lookupDomain()
+		time.Sleep(waitDuration)
+	}
+}
+
+func makeHTTPRequest() {
+	if url == "" {
+		return
+	}
+
+	// Create a new client so that the connection won't be shared with other requests.
+	client := http.Client{
+		CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+	}
+	start := time.Now()
+	resp, err := client.Head(url)
+	if err != nil {
+		log.Errorf("http request failed after %s: %s", time.Since(start).Round(time.Millisecond), err)
+		return
+	}
+	defer resp.Body.Close()
+
+	log.Infof("http response after %s: %d", time.Since(start).Round(time.Millisecond), resp.StatusCode)
+}
+
+func lookupDomain() {
+	if lookup == "" {
+		return
+	}
+
+	// Create DNS query.
+	dnsQuery := new(dns.Msg)
+	dnsQuery.SetQuestion(dns.Fqdn(lookup), dns.TypeA)
+
+	// Send request.
+	start := time.Now()
+	reply, err := dns.Exchange(dnsQuery, dnsResolver)
+	if err != nil {
+		log.Errorf("dns request failed after %s: %s", time.Since(start).Round(time.Millisecond), err)
+		return
+	}
+
+	log.Infof("dns response after %s: %s", time.Since(start).Round(time.Millisecond), dns.RcodeToString[reply.Rcode])
+}

cmds/trafficgen/pack

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+baseDir="$( cd "$(dirname "$0")" && pwd )"
+cd "$baseDir"
+
+echo "building..."
+GOOS=windows go build
+
+dstFile="../../../portmaster-windows-kext/install/MINGW/amd64/trafficgen.exe"
+if [[ -d $(dirname "$dstFile") ]]; then
+  # make path absolute
+  dstFile="$(cd "$(dirname "$dstFile")" && pwd)/$(basename "$dstFile")"
+  # copy
+  echo "copying to $dstFile"
+  cp trafficgen.exe "$dstFile"
+else
+  echo "not copying to $dstFile (dir does not exist)"
+fi

cmds/updatemgr/main.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"context"
 	"os"
 	"path/filepath"
 
@@ -22,7 +23,34 @@ var rootCmd = &cobra.Command{
 		}
 
 		registry = &updater.ResourceRegistry{}
-		return registry.Initialize(utils.NewDirStructure(absPath, 0o755))
+		err = registry.Initialize(utils.NewDirStructure(absPath, 0o755))
+		if err != nil {
+			return err
+		}
+
+		registry.AddIndex(updater.Index{
+			Path:   "stable.json",
+			Stable: true,
+			Beta:   false,
+		})
+
+		registry.AddIndex(updater.Index{
+			Path:   "beta.json",
+			Stable: false,
+			Beta:   true,
+		})
+
+		err = registry.LoadIndexes(context.TODO())
+		if err != nil {
+			return err
+		}
+
+		err = registry.ScanStorage("")
+		if err != nil {
+			return err
+		}
+
+		return nil
 	},
 	SilenceUsage: true,
 }

cmds/updatemgr/purge.go

@@ -1,13 +1,11 @@
 package main
 
 import (
-	"context"
 	"fmt"
 
 	"github.com/spf13/cobra"
 
 	"github.com/safing/portbase/log"
-	"github.com/safing/portbase/updater"
 )
 
 func init() {
@@ -29,28 +27,6 @@ func purge(cmd *cobra.Command, args []string) error {
 	}
 	defer log.Shutdown()
 
-	registry.AddIndex(updater.Index{
-		Path:   "stable.json",
-		Stable: true,
-		Beta:   false,
-	})
-
-	registry.AddIndex(updater.Index{
-		Path:   "beta.json",
-		Stable: false,
-		Beta:   true,
-	})
-
-	err = registry.LoadIndexes(context.TODO())
-	if err != nil {
-		return err
-	}
-
-	err = scanStorage()
-	if err != nil {
-		return err
-	}
-
 	registry.SelectVersions()
 	registry.Purge(3)
 

cmds/updatemgr/scan.go

@@ -3,9 +3,6 @@ package main
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"path/filepath"
-	"strings"
 
 	"github.com/spf13/cobra"
 )
@@ -22,48 +19,21 @@ var scanCmd = &cobra.Command{
 }
 
 func scan(cmd *cobra.Command, args []string) error {
-	err := scanStorage()
+	// Reset and rescan.
+	registry.Reset()
+	err := registry.ScanStorage("")
 	if err != nil {
 		return err
 	}
 
-	// export beta
+	// Export latest versions.
-	data, err := json.MarshalIndent(exportSelected(true), "", " ")
+	data, err := json.MarshalIndent(exportSelected(false), "", " ")
 	if err != nil {
 		return err
 	}
-	// print
+	// Print them.
-	fmt.Println("beta:")
 	fmt.Println(string(data))
 
-	// export stable
-	data, err = json.MarshalIndent(exportSelected(false), "", " ")
-	if err != nil {
-		return err
-	}
-	// print
-	fmt.Println("\nstable:")
-	fmt.Println(string(data))
-
-	return nil
-}
-
-func scanStorage() error {
-	files, err := ioutil.ReadDir(registry.StorageDir().Path)
-	if err != nil {
-		return err
-	}
-
-	// scan "all" and all "os_platform" dirs
-	for _, file := range files {
-		if file.IsDir() && (file.Name() == "all" || strings.Contains(file.Name(), "_")) {
-			err := registry.ScanStorage(filepath.Join(registry.StorageDir().Path, file.Name()))
-			if err != nil {
-				return err
-			}
-		}
-	}
-
 	return nil
 }
 

cmds/updatemgr/staging.go

@@ -1,14 +1,12 @@
 package main
 
 import (
-	"context"
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"os"
 	"path/filepath"
 
-	"github.com/safing/portbase/updater"
 	"github.com/spf13/cobra"
 )
 
@@ -29,37 +27,14 @@ var stageCmd = &cobra.Command{
 }
 
 func stage(cmd *cobra.Command, args []string) error {
-	registry.AddIndex(updater.Index{
-		Path:   "stable.json",
-		Stable: true,
-		Beta:   false,
-	})
-
-	registry.AddIndex(updater.Index{
-		Path:   "beta.json",
-		Stable: false,
-		Beta:   true,
-	})
-
-	err := registry.LoadIndexes(context.TODO())
-	if err != nil {
-		return err
-	}
-
-	err = scanStorage()
-	if err != nil {
-		return err
-	}
-
 	// Check if we want to reset staging instead.
 	if stageReset {
 		for _, stagedPath := range exportStaging(true) {
-			err = os.Remove(stagedPath)
+			err := os.Remove(stagedPath)
 			if err != nil {
 				return err
 			}
 		}
-
 		return nil
 	}
 

cmds/updatemgr/update.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"path/filepath"
+	"sort"
 
 	"github.com/spf13/cobra"
 )
@@ -21,10 +22,8 @@ var updateCmd = &cobra.Command{
 }
 
 func update(cmd *cobra.Command, args []string) error {
-	err := scanStorage()
+	// Set stable and beta to latest version.
-	if err != nil {
+	updateToLatestVersion(true, true)
-		return err
-	}
 
 	// Export versions.
 	betaData, err := json.MarshalIndent(exportSelected(true), "", " ")
@@ -75,3 +74,14 @@ func update(cmd *cobra.Command, args []string) error {
 
 	return nil
 }
+
+func updateToLatestVersion(stable, beta bool) error {
+	for _, resource := range registry.Export() {
+		sort.Sort(resource)
+		err := resource.AddVersion(resource.Versions[0].VersionNumber, false, stable, beta)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

cmds/winkext-test/main.go

@@ -0,0 +1,175 @@
+// +build windows
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"syscall"
+
+	"github.com/safing/portbase/log"
+	"github.com/safing/portmaster/firewall/interception/windowskext"
+	"github.com/safing/portmaster/network/packet"
+)
+
+var (
+	packets    chan packet.Packet
+	shutdownCh = make(chan struct{})
+
+	getPayload      bool
+	rerouteDNS      bool
+	permanentAccept bool
+	maxPackets      int
+)
+
+func init() {
+	flag.BoolVar(&getPayload, "get-payload", false, "get payload of handled packets")
+	flag.BoolVar(&rerouteDNS, "reroute-dns", false, "reroute dns to own IP")
+	flag.BoolVar(&permanentAccept, "permanent-accept", false, "permanent-accept packets")
+	flag.IntVar(&maxPackets, "max-packets", 0, "handle specified amount of packets, then exit")
+}
+
+func main() {
+	flag.Parse()
+
+	// check parameter count
+	if flag.NArg() != 2 {
+		fmt.Printf("usage: %s [options] <dll> <sys>\n", os.Args[0])
+		flag.Usage()
+		os.Exit(1)
+	}
+
+	// logging
+	err := log.Start()
+	if err != nil {
+		fmt.Printf("failed to start logging: %s\n", err)
+		os.Exit(1)
+	}
+	defer log.Shutdown()
+	log.SetLogLevel(log.TraceLevel)
+	log.Info("starting windows kext test program")
+
+	// Check paths.
+	dllPath, err := filepath.Abs(flag.Arg(0))
+	if err == nil {
+		_, err = os.Stat(dllPath)
+	}
+	if err != nil {
+		log.Criticalf("cannot find .dll: %s\n", err)
+		return
+	}
+	log.Infof("using .dll at %s", dllPath)
+
+	sysPath, err := filepath.Abs(flag.Arg(1))
+	if err == nil {
+		_, err = os.Stat(sysPath)
+	}
+	if err != nil {
+		log.Criticalf("cannot find .sys: %s", err)
+		return
+	}
+	log.Infof("using .sys at %s", sysPath)
+
+	// init
+	err = windowskext.Init(dllPath, sysPath)
+	if err != nil {
+		log.Criticalf("failed to init kext: %s", err)
+		return
+	}
+
+	// start
+	err = windowskext.Start()
+	if err != nil {
+		log.Criticalf("failed to start kext: %s", err)
+		return
+	}
+
+	packets = make(chan packet.Packet, 1000)
+	go windowskext.Handler(packets)
+	go handlePackets()
+
+	// catch interrupt for clean shutdown
+	signalCh := make(chan os.Signal, 1)
+	signal.Notify(
+		signalCh,
+		os.Interrupt,
+		syscall.SIGHUP,
+		syscall.SIGINT,
+		syscall.SIGTERM,
+		syscall.SIGQUIT,
+	)
+	select {
+	case <-signalCh:
+		fmt.Println(" <INTERRUPT>")
+		log.Warning("program was interrupted, shutting down")
+	case <-shutdownCh:
+		log.Warningf("shutting down")
+	}
+
+	// stop
+	err = windowskext.Stop()
+	if err != nil {
+		log.Criticalf("failed to stop kext: %s", err)
+	}
+
+	log.Info("shutdown complete")
+}
+
+func handlePackets() {
+	var err error
+	var handledPackets int
+
+	for {
+		pkt := <-packets
+
+		if pkt == nil {
+			log.Infof("stopped handling packets")
+			return
+		}
+
+		log.Infof("received packet: %s", pkt)
+		handledPackets++
+
+		if getPayload {
+			data, err := pkt.GetPayload()
+			if err != nil {
+				log.Errorf("failed to get payload: %s", err)
+			} else {
+				log.Infof("payload is: %x", data)
+			}
+		}
+
+		// reroute dns requests to nameserver
+		if rerouteDNS {
+			if pkt.IsOutbound() && !pkt.Info().Src.Equal(pkt.Info().Dst) && pkt.Info().DstPort == 53 {
+				log.Infof("rerouting %s", pkt)
+				err = pkt.RerouteToNameserver()
+				if err != nil {
+					log.Errorf("failed to reroute: %s", err)
+				}
+				continue
+			}
+		}
+
+		// accept all
+		log.Infof("accepting %s", pkt)
+		if permanentAccept {
+			err = pkt.PermanentAccept()
+		} else {
+			err = pkt.Accept()
+		}
+		if err != nil {
+			log.Errorf("failed to accept: %s", err)
+		}
+
+		if maxPackets > 0 && handledPackets > maxPackets {
+			log.Infof("max-packets (%d) reached", maxPackets)
+			close(shutdownCh)
+			return
+		}
+
+	}
+}

cmds/winkext-test/pack

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+baseDir="$( cd "$(dirname "$0")" && pwd )"
+cd "$baseDir"
+
+echo "building..."
+GOOS=windows go build
+
+dstFile="../../../portmaster-windows-kext/install/MINGW/amd64/winkext-test.exe"
+if [[ -d $(dirname "$dstFile") ]]; then
+  # make path absolute
+  dstFile="$(cd "$(dirname "$dstFile")" && pwd)/$(basename "$dstFile")"
+  # copy
+  echo "copying to $dstFile"
+  cp winkext-test.exe "$dstFile"
+else
+  echo "not copying to $dstFile (dir does not exist)"
+fi

firewall/interception/windowskext/test/main.go

@@ -1,118 +0,0 @@
-// +build windows
-
-package main
-
-import (
-	"fmt"
-	"os"
-	"os/signal"
-	"syscall"
-
-	"github.com/safing/portbase/log"
-	"github.com/safing/portmaster/firewall/interception/windowskext"
-	"github.com/safing/portmaster/network/packet"
-)
-
-var (
-	packets chan packet.Packet
-)
-
-func main() {
-
-	// check parameter count
-	if len(os.Args) < 3 {
-		fmt.Printf("usage: %s <dll> <sys>", os.Args[0])
-		os.Exit(1)
-	}
-
-	// check parameters
-	for i := 1; i < 3; i++ {
-		if _, err := os.Stat(os.Args[i]); err != nil {
-			fmt.Printf("could not access %s: %s", os.Args[i], err)
-			os.Exit(2)
-		}
-	}
-
-	// logging
-	_ = log.Start()
-	log.Info("starting Portmaster Windows Kext Test Program")
-
-	// init
-	err := windowskext.Init(os.Args[1], os.Args[2])
-	if err != nil {
-		panic(err)
-	}
-
-	// start
-	err = windowskext.Start()
-	if err != nil {
-		panic(err)
-	}
-
-	packets = make(chan packet.Packet, 1000)
-	go windowskext.Handler(packets)
-	go handlePackets()
-
-	// catch interrupt for clean shutdown
-	signalCh := make(chan os.Signal, 1)
-	signal.Notify(
-		signalCh,
-		os.Interrupt,
-		syscall.SIGHUP,
-		syscall.SIGINT,
-		syscall.SIGTERM,
-		syscall.SIGQUIT,
-	)
-	<-signalCh
-	fmt.Println(" <INTERRUPT>")
-	log.Warning("program was interrupted, shutting down")
-
-	// stop
-	err = windowskext.Stop()
-	if err != nil {
-		panic(err)
-	}
-
-	log.Info("shutdown complete")
-	log.Shutdown()
-
-	os.Exit(0)
-}
-
-func handlePackets() {
-	for {
-		pkt := <-packets
-
-		if pkt == nil {
-			log.Infof("stopped handling packets")
-			return
-		}
-
-		log.Infof("received packet: %s", pkt)
-
-		data, err := pkt.GetPayload()
-		if err != nil {
-			log.Errorf("failed to get payload: %s", err)
-		} else {
-			log.Infof("payload is: %x", data)
-		}
-
-		// reroute dns requests to nameserver
-		if pkt.IsOutbound() && !pkt.Info().Src.Equal(pkt.Info().Dst) && pkt.Info().DstPort == 53 {
-			log.Infof("rerouting %s", pkt)
-			err = pkt.RerouteToNameserver()
-			if err != nil {
-				log.Errorf("failed to reroute: %s", err)
-			}
-			continue
-		}
-
-		// accept all
-		log.Infof("accepting %s", pkt)
-		err = pkt.PermanentAccept()
-		if err != nil {
-			log.Errorf("failed to accept: %s", err)
-		}
-
-	}
-}