tun: land gen2 follow-up commits missed by #256 squash (#258)

src/config/validation.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/daniellavrushin/b4/engine"
 	"github.com/daniellavrushin/b4/log"
 	"github.com/daniellavrushin/b4/utils"
 )
@@ -255,9 +256,18 @@ func (c *Config) Validate() error {
 		return v.result()
 	}
 
-	if c.Queue.Mode == "tun" && c.Queue.TUN.OutInterface == "" {
-		v.add("queue.tun.out_interface", "required", "tun out_interface is required in TUN mode (e.g. eth0, wan0, l2tp-vpn)", nil)
-		return v.result()
+	if c.Queue.Mode == "tun" {
+		if c.Queue.TUN.OutInterface == "" {
+			v.add("queue.tun.out_interface", "required", "tun out_interface is required in TUN mode (e.g. eth0, wan0, l2tp-vpn)", nil)
+			return v.result()
+		}
+		const tunReservedMarkBits = uint(engine.TunSteerMark | engine.TunClientMark | engine.ReinjectMarkBit)
+		if m := c.MainInjectedMark(); m&tunReservedMarkBits != 0 {
+			v.addf("queue.mark", "mark_conflict", map[string]any{"mark": fmt.Sprintf("0x%x", m)},
+				"queue mark 0x%x overlaps reserved TUN mark bits (0x%x steer, 0x%x client, 0x%x reinject); choose a mark clear of those bits",
+				m, uint(engine.TunSteerMark), uint(engine.TunClientMark), uint(engine.ReinjectMarkBit))
+			return v.result()
+		}
 	}
 
 	if c.Queue.StartNum < 0 || c.Queue.StartNum > 65535 {

src/config/validation_test.go

@@ -4,7 +4,10 @@ import (
 	"errors"
 	"os"
 	"path/filepath"
+	"strings"
 	"testing"
+
+	"github.com/daniellavrushin/b4/engine"
 )
 
 func mustValidationErr(t *testing.T, err error) *ValidationError {
@@ -337,6 +340,21 @@ func TestValidate_QueueFields(t *testing.T) {
 			t.Errorf("valid tun config rejected: %v", err)
 		}
 	})
+
+	t.Run("tun mode rejects mark overlapping reserved bits", func(t *testing.T) {
+		cfg := NewConfig()
+		cfg.Queue.Mode = "tun"
+		cfg.Queue.TUN.OutInterface = "eth0"
+		cfg.Queue.Mark = engine.TunSteerMark
+		ve := mustValidationErr(t, cfg.Validate())
+		f := findField(ve, "queue.mark", "mark_conflict")
+		if f == nil {
+			t.Fatalf("missing queue.mark mark_conflict; got %+v", ve.Fields)
+		}
+		if !strings.Contains(f.Message, "reserved TUN mark bits") {
+			t.Errorf("expected reserved-TUN-bits message, got %q", f.Message)
+		}
+	})
 }
 
 func TestValidate_RequiredSetID(t *testing.T) {

src/engine/engine.go

@@ -1,6 +1,10 @@
 package engine
 
-const ReinjectMarkBit = 0x800000
+const (
+	TunSteerMark    = 0x80000
+	TunClientMark   = 0x100000
+	ReinjectMarkBit = 0x800000
+)
 
 type PacketVerdict int
 

src/main.go

@@ -199,8 +199,12 @@ func runB4(cmd *cobra.Command, args []string) error {
 
 	if clearTables {
 		log.Infof("Clearing iptables rules as requested (--clear-iptables)")
-		tables.ClearRules(&cfg)
+		clearErr := tables.ClearRules(&cfg)
 		tables.RoutingClearAll()
+		b4tun.ClearStaleArtifacts(&cfg)
+		if clearErr != nil {
+			return fmt.Errorf("failed to clear iptables/nftables rules: %w", clearErr)
+		}
 		log.Infof("IPTables rules cleared successfully")
 		return nil
 	}
@@ -244,6 +248,10 @@ func runB4(cmd *cobra.Command, args []string) error {
 			cfg.Queue.TUN.DeviceName, cfg.Queue.TUN.OutInterface, cfg.Queue.Threads)
 
 		if !cfg.System.Tables.SkipSetup {
+			log.Tracef("Clearing any pre-existing NFQUEUE/tables rules before TUN setup")
+			if err := tables.ClearRules(&cfg); err != nil {
+				log.Warnf("TUN: failed to clear pre-existing tables rules (continuing): %v", err)
+			}
 			if err := tables.ApplyMasqueradeOnly(&cfg); err != nil {
 				metrics.RecordEvent("error", fmt.Sprintf("Failed to apply masquerade: %v", err))
 				return fmt.Errorf("failed to apply masquerade: %w", err)
@@ -286,6 +294,7 @@ func runB4(cmd *cobra.Command, args []string) error {
 		if !cfg.System.Tables.SkipSetup {
 			log.Tracef("Clearing existing iptables/nftables rules")
 			tables.ClearRules(&cfg)
+			b4tun.ClearStaleArtifacts(&cfg)
 
 			log.Tracef("Adding tables rules")
 			if err := tables.AddRules(&cfg); err != nil {

src/nfq/verdict.go

@@ -51,7 +51,7 @@ func (w *Worker) InitSender() error {
 	}
 	w.sock = s
 	if device != "" {
-		cs, err := sock.NewSenderWithMark(0)
+		cs, err := sock.NewSenderWithMark(engine.TunClientMark)
 		if err != nil {
 			w.sock.Close()
 			w.sock = nil

src/tun/capture.go

@@ -5,6 +5,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/daniellavrushin/b4/engine"
 	"github.com/daniellavrushin/b4/log"
 	"github.com/daniellavrushin/b4/tables"
 )
@@ -12,8 +13,8 @@ import (
 const (
 	tunCaptureChain   = "B4_TUN"
 	tunProbeChain     = "B4_TUN_PROBE"
-	defaultSteerMark  = 0x80000
-	defaultClientMark = 0x100000
+	defaultSteerMark  = engine.TunSteerMark
+	defaultClientMark = engine.TunClientMark
 	captureRulePrio   = 90
 )
 

src/tun/cleanup.go

@@ -0,0 +1,134 @@
+package tun
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/daniellavrushin/b4/config"
+	"github.com/daniellavrushin/b4/log"
+)
+
+func ClearStaleArtifacts(cfg *config.Config) {
+	device := cfg.Queue.TUN.DeviceName
+	if device == "" {
+		device = defaultDeviceName
+	}
+	routeTable := cfg.Queue.TUN.RouteTable
+	if routeTable == 0 {
+		routeTable = defaultRouteTable
+	}
+	captureTable := routeTable - 1
+	if captureTable <= 0 {
+		captureTable = routeTable + 1
+	}
+
+	cleared := false
+
+	for _, base := range []string{"PREROUTING", "OUTPUT"} {
+		for {
+			if _, err := run("iptables", "-t", "mangle", "-D", base, "-j", tunCaptureChain); err != nil {
+				break
+			}
+			cleared = true
+		}
+	}
+	run("iptables", "-t", "mangle", "-F", tunCaptureChain)
+	run("iptables", "-t", "mangle", "-X", tunCaptureChain)
+
+	for {
+		if _, err := run("iptables", "-t", "raw", "-D", "OUTPUT", "-m", "mark", "--mark", reinjectMarkMatch(), "-j", "CT", "--notrack"); err != nil {
+			break
+		}
+		cleared = true
+	}
+
+	for _, dir := range []string{"-i", "-o"} {
+		for {
+			if _, err := run("iptables", "-D", "FORWARD", dir, device, "-j", "ACCEPT"); err != nil {
+				break
+			}
+			cleared = true
+		}
+	}
+
+	if clearTunSNAT(device) {
+		cleared = true
+	}
+
+	if clearOwnedRoutingTable(steerMarkMatch(), captureTable) {
+		cleared = true
+	}
+	if clearOwnedRoutingTable(fmt.Sprintf("0x%x", cfg.MainInjectedMark()), routeTable) {
+		cleared = true
+	}
+
+	if interfaceExists(device) && isTunDevice(device) {
+		run("ip", "link", "del", device)
+		cleared = true
+	}
+
+	if cleared {
+		log.Infof("TUN: cleared stale TUN-engine artifacts left by a previous run")
+	}
+}
+
+func steerMarkMatch() string {
+	return fmt.Sprintf("0x%x/0x%x", defaultSteerMark, defaultSteerMark)
+}
+
+func clearTunSNAT(device string) bool {
+	out, err := run("iptables", "-t", "nat", "-S", "POSTROUTING")
+	if err != nil {
+		return false
+	}
+	cleared := false
+	for _, line := range strings.Split(out, "\n") {
+		line = strings.TrimSpace(line)
+		if !strings.HasPrefix(line, "-A POSTROUTING") {
+			continue
+		}
+		if ruleFieldValue(line, "-o") != device || ruleFieldValue(line, "-j") != "SNAT" {
+			continue
+		}
+		spec := strings.Fields(strings.TrimPrefix(line, "-A POSTROUTING"))
+		if _, err := run(append([]string{"iptables", "-t", "nat", "-D", "POSTROUTING"}, spec...)...); err == nil {
+			cleared = true
+		}
+	}
+	return cleared
+}
+
+func clearOwnedRoutingTable(fwmark string, table int) bool {
+	tableStr := strconv.Itoa(table)
+	out, err := run("ip", "rule", "show")
+	if err != nil {
+		return false
+	}
+	bare := strings.SplitN(fwmark, "/", 2)[0]
+	marks := make(map[string]struct{})
+	for _, line := range strings.Split(out, "\n") {
+		if ruleFieldValue(line, "lookup") != tableStr {
+			continue
+		}
+		fw := ruleFieldValue(line, "fwmark")
+		if fw == "" {
+			continue
+		}
+		if fw == fwmark || fw == bare || strings.HasPrefix(fw, bare+"/") {
+			marks[fw] = struct{}{}
+		}
+	}
+	if len(marks) == 0 {
+		return false
+	}
+	run("ip", "route", "flush", "table", tableStr)
+	for fw := range marks {
+		for {
+			if _, err := run("ip", "rule", "del", "fwmark", fw, "lookup", tableStr); err != nil {
+				break
+			}
+		}
+	}
+	return true
+}