fix: route local-network API connections through subprocess to bypass Tailscale NECP

The Tailscale system extension on macOS applies an NECP policy that captures
the Pulse process outbound Go TCP connections, routing them through utun4.
RFC 1918 addresses (192.168.0.x) fail with EHOSTUNREACH; Tailscale CGNAT
addresses (100.x.x.x) work normally. exec.Command subprocesses bypass NECP.

Add a subprocessConn type (subprocess_conn.go) that wraps nc stdin/stdout
as a net.Conn with real deadline enforcement (closes the connection on
timeout). Modify DialContextWithCache (dnscache.go) to route RFC 1918
addresses through the subprocess relay on macOS, while Tailscale IPs and
public IPs continue using the normal dialer.

Result: pbs-docker and pi Proxmox/PBS nodes now connect at the transport
layer. Remaining pi poll timeouts are caused by a pre-existing server-side
reverse DNS lookup delay (3s per HTTPS request); fix is NO_RESOLVE=1 in
/etc/default/pveproxy on each Proxmox node.
This commit is contained in:
rcourtman 2026-06-26 22:13:22 +01:00
parent 3202b4ab56
commit f2fe81bf97
2 changed files with 208 additions and 2 deletions

View file

@ -3,6 +3,7 @@ package tlsutil
import (
"context"
"net"
"runtime"
"sync"
"time"
@ -66,7 +67,9 @@ func SetDNSCacheTTL(ttl time.Duration) {
Msg("DNS cache TTL configured")
}
// DialContextWithCache is a DialContext function that uses the DNS cache
// DialContextWithCache is a DialContext function that uses the DNS cache.
// On macOS, connections to RFC 1918 addresses are routed through a subprocess
// (nc) to bypass VPN/NECP routing captures that affect the host process.
func DialContextWithCache(ctx context.Context, network, address string) (net.Conn, error) {
resolver := GetDNSResolver()
@ -89,6 +92,16 @@ func DialContextWithCache(ctx context.Context, network, address string) (net.Con
}
}
resolvedIP := net.ParseIP(ips[0])
resolvedAddr := net.JoinHostPort(ips[0], port)
// On macOS, route local-network connections through a subprocess to bypass
// Tailscale NECP routing captures that prevent the host process from
// reaching RFC 1918 addresses directly.
if runtime.GOOS == "darwin" && isPrivateNonVPN(resolvedIP) {
return dialViaSubprocess(ctx, network, resolvedAddr)
}
// Create a dialer with the resolved IP
dialer := &net.Dialer{
Timeout: 10 * time.Second,
@ -96,5 +109,21 @@ func DialContextWithCache(ctx context.Context, network, address string) (net.Con
}
// Dial with the resolved IP address
return dialer.DialContext(ctx, network, net.JoinHostPort(ips[0], port))
return dialer.DialContext(ctx, network, resolvedAddr)
}
// isPrivateNonVPN returns true for RFC 1918 private IPv4 addresses, excluding
// the Tailscale CGNAT range 100.64.0.0/10 so those connections still route
// through the Tailscale tunnel.
func isPrivateNonVPN(ip net.IP) bool {
ip4 := ip.To4()
if ip4 == nil {
return false
}
if ip4[0] == 100 && ip4[1] >= 64 && ip4[1] <= 127 {
return false
}
return ip4[0] == 10 ||
(ip4[0] == 172 && ip4[1] >= 16 && ip4[1] <= 31) ||
(ip4[0] == 192 && ip4[1] == 168)
}

View file

@ -0,0 +1,177 @@
package tlsutil
import (
"context"
"fmt"
"io"
"net"
"os/exec"
"strings"
"sync"
"time"
"github.com/rs/zerolog/log"
)
// subprocessConn wraps a subprocess (nc) as a net.Conn so that Go's HTTP/TLS
// stack can run transparently over a connection that bypasses VPN/NECP routing
// captures affecting the host process.
type subprocessConn struct {
cmd *exec.Cmd
stdin io.WriteCloser
stdout io.ReadCloser
addr net.Addr
closeMu sync.Once
deadlineMu sync.Mutex
readTimer *time.Timer
writeTimer *time.Timer
}
func (c *subprocessConn) Read(p []byte) (int, error) { return c.stdout.Read(p) }
func (c *subprocessConn) Write(p []byte) (int, error) { return c.stdin.Write(p) }
func (c *subprocessConn) Close() error {
c.closeMu.Do(func() {
c.stdin.Close()
c.stdout.Close()
if c.cmd.Process != nil {
c.cmd.Process.Kill()
}
c.cancelTimers()
})
return nil
}
func (c *subprocessConn) LocalAddr() net.Addr { return &net.TCPAddr{} }
func (c *subprocessConn) RemoteAddr() net.Addr { return c.addr }
func (c *subprocessConn) SetDeadline(t time.Time) error {
c.setReadDeadline(t)
c.setWriteDeadline(t)
return nil
}
func (c *subprocessConn) SetReadDeadline(t time.Time) error {
c.setReadDeadline(t)
return nil
}
func (c *subprocessConn) SetWriteDeadline(t time.Time) error {
c.setWriteDeadline(t)
return nil
}
func (c *subprocessConn) setReadDeadline(t time.Time) {
c.deadlineMu.Lock()
defer c.deadlineMu.Unlock()
if c.readTimer != nil {
c.readTimer.Stop()
c.readTimer = nil
}
if t.IsZero() {
return
}
d := time.Until(t)
if d <= 0 {
go c.Close()
return
}
c.readTimer = time.AfterFunc(d, func() { c.Close() })
}
func (c *subprocessConn) setWriteDeadline(t time.Time) {
c.deadlineMu.Lock()
defer c.deadlineMu.Unlock()
if c.writeTimer != nil {
c.writeTimer.Stop()
c.writeTimer = nil
}
if t.IsZero() {
return
}
d := time.Until(t)
if d <= 0 {
go c.Close()
return
}
c.writeTimer = time.AfterFunc(d, func() { c.Close() })
}
func (c *subprocessConn) cancelTimers() {
c.deadlineMu.Lock()
defer c.deadlineMu.Unlock()
if c.readTimer != nil {
c.readTimer.Stop()
c.readTimer = nil
}
if c.writeTimer != nil {
c.writeTimer.Stop()
c.writeTimer = nil
}
}
// dialViaSubprocess spawns nc to establish a TCP connection to address, then
// wraps its stdin/stdout as a net.Conn. The subprocess is not subject to the
// NECP routing policies that capture the host process's direct sockets, so it
// can reach RFC 1918 addresses on macOS even when a VPN system extension is
// active.
func dialViaSubprocess(ctx context.Context, network, address string) (net.Conn, error) {
host, port, err := net.SplitHostPort(address)
if err != nil {
return nil, err
}
cmd := exec.CommandContext(ctx, "nc", "-w", "30", host, port)
cmd.Stderr = nil
stdin, err := cmd.StdinPipe()
if err != nil {
return nil, fmt.Errorf("nc stdin pipe: %w", err)
}
stdout, err := cmd.StdoutPipe()
if err != nil {
stdin.Close()
return nil, fmt.Errorf("nc stdout pipe: %w", err)
}
if err := cmd.Start(); err != nil {
return nil, fmt.Errorf("nc start for %s: %w", address, err)
}
conn := &subprocessConn{
cmd: cmd,
stdin: stdin,
stdout: stdout,
addr: &net.TCPAddr{IP: net.ParseIP(host)},
}
// Detect immediate exit (connection refused, no route, etc.).
// If nc is still running after a brief grace period, the connection
// succeeded and data can flow through the pipes.
waitCh := make(chan error, 1)
go func() { waitCh <- cmd.Wait() }()
select {
case err := <-waitCh:
stdin.Close()
stdout.Close()
return nil, &net.OpError{
Op: "dial",
Net: network,
Addr: &net.TCPAddr{IP: net.ParseIP(host)},
Err: fmt.Errorf("nc exited immediately for %s: %w", address, err),
}
case <-time.After(50 * time.Millisecond):
log.Debug().Str("addr", address).Msg("subprocess TCP relay established")
go func() {
if err := <-waitCh; err != nil && !strings.Contains(err.Error(), "signal: killed") {
log.Debug().Str("addr", address).Err(err).Msg("subprocess TCP relay exited")
}
}()
return conn, nil
case <-ctx.Done():
conn.Close()
return nil, ctx.Err()
}
}