vikarti.anatra/safing-portmaster
1
0
Fork 0
mirror of https://github.com/safing/portmaster synced 2025-09-02 02:29:12 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge pull request #821 from safing/feature/spn-improvements

Browse source 
SPN Routing Improvements (and housekeeping)
This commit is contained in:
Daniel 2022-08-30 17:18:33 +02:00 • committed by GitHub
commit 636c0dc228
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
32 changed files with 255 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
.github/workflows/go.yml vendored
View file

@ -20,14 +20,14 @@ jobs:
- uses: actions/setup-go@v3 - uses: actions/setup-go@v3
with: with:
go-version: '^1.18' go-version: '^1.19'
- name: Run golangci-lint - name: Run golangci-lint
uses: golangci/golangci-lint-action@v3 uses: golangci/golangci-lint-action@v3
with: with:
version: v1.45.1 version: v1.49.0
only-new-issues: true only-new-issues: true
args: -c ./.golangci.yml args: -c ./.golangci.yml --timeout 15m
- name: Get dependencies - name: Get dependencies
run: go mod download run: go mod download
@ -45,7 +45,7 @@ jobs:
- name: Setup Go - name: Setup Go
uses: actions/setup-go@v3 uses: actions/setup-go@v3
with: with:
go-version: '^1.18' go-version: '^1.19'
- name: Get dependencies - name: Get dependencies
run: go mod download run: go mod download

6
.golangci.yml
View file

@ -8,6 +8,7 @@ linters:
- contextcheck - contextcheck
- cyclop - cyclop
- exhaustivestruct - exhaustivestruct
- exhaustruct
- forbidigo - forbidigo
- funlen - funlen
- gochecknoglobals - gochecknoglobals
@ -17,6 +18,7 @@ linters:
- goerr113 - goerr113
- gomnd - gomnd
- ifshort - ifshort
- interfacebloat
- interfacer - interfacer
- ireturn - ireturn
- lll - lll
@ -24,6 +26,9 @@ linters:
- nilnil - nilnil
- nlreturn - nlreturn
- noctx - noctx
- nolintlint
- nonamedreturns
- nosnakecase
- revive - revive
- tagliatelle - tagliatelle
- testpackage - testpackage
@ -31,7 +36,6 @@ linters:
- whitespace - whitespace
- wrapcheck - wrapcheck
- wsl - wsl
- nolintlint
linters-settings: linters-settings:
revive: revive:

3
cmds/portmaster-core/main.go
View file

@ -1,6 +1,7 @@
//nolint:gci,nolintlint
package main package main
import ( //nolint:gci,nolintlint import (
"os" "os"
"github.com/safing/portbase/info" "github.com/safing/portbase/info"

17
core/pmtesting/testing.go
View file

@ -2,18 +2,17 @@
// //
// Usage: // Usage:
// //
// package name // package name
// //
// import ( // import (
// "testing" // "testing"
// //
// "github.com/safing/portmaster/core/pmtesting" // "github.com/safing/portmaster/core/pmtesting"
// ) // )
//
// func TestMain(m *testing.M) {
// pmtesting.TestMain(m, module)
// }
// //
// func TestMain(m *testing.M) {
// pmtesting.TestMain(m, module)
// }
package pmtesting package pmtesting
import ( import (

21
firewall/config.go
View file

@ -1,10 +1,9 @@
package firewall package firewall
import ( import (
"github.com/safing/portbase/api"
"github.com/safing/portbase/config" "github.com/safing/portbase/config"
"github.com/safing/portbase/notifications" "github.com/safing/portbase/notifications"
"github.com/safing/portmaster/core" "github.com/safing/spn/captain"
) )
// Configuration Keys. // Configuration Keys.
@ -26,9 +25,6 @@ var (
CfgOptionDNSQueryInterceptionKey = "filter/dnsQueryInterception" CfgOptionDNSQueryInterceptionKey = "filter/dnsQueryInterception"
cfgOptionDNSQueryInterceptionOrder = 97 cfgOptionDNSQueryInterceptionOrder = 97
dnsQueryInterception config.BoolOption dnsQueryInterception config.BoolOption
devMode config.BoolOption
apiListenAddress config.StringOption
) )
func registerConfig() error { func registerConfig() error {
@ -108,8 +104,17 @@ func registerConfig() error {
} }
askTimeout = config.Concurrent.GetAsInt(CfgOptionAskTimeoutKey, 60) askTimeout = config.Concurrent.GetAsInt(CfgOptionAskTimeoutKey, 60)
devMode = config.Concurrent.GetAsBool(core.CfgDevModeKey, false)
apiListenAddress = config.GetAsString(api.CfgDefaultListenAddressKey, "")
return nil return nil
} }
var (
filterEnabled config.BoolOption
tunnelEnabled config.BoolOption
useCommunityNodes config.BoolOption
)
func getConfig() {
filterEnabled = config.Concurrent.GetAsBool(CfgOptionEnableFilterKey, true)
tunnelEnabled = config.Concurrent.GetAsBool(captain.CfgOptionEnableSPNKey, false)
useCommunityNodes = config.Concurrent.GetAsBool(captain.CfgOptionUseCommunityNodesKey, true)
}

23
firewall/filter.go
View file

@ -2,22 +2,12 @@ package firewall
import ( import (
"github.com/safing/portbase/config" "github.com/safing/portbase/config"
"github.com/safing/portbase/log"
"github.com/safing/portbase/modules" "github.com/safing/portbase/modules"
"github.com/safing/portbase/modules/subsystems" "github.com/safing/portbase/modules/subsystems"
_ "github.com/safing/portmaster/core" _ "github.com/safing/portmaster/core"
"github.com/safing/portmaster/intel/filterlists"
"github.com/safing/spn/captain"
) )
var ( var filterModule *modules.Module
filterModule *modules.Module
filterEnabled config.BoolOption
tunnelEnabled config.BoolOption
unbreakFilterListIDs = []string{"UNBREAK"}
resolvedUnbreakFilterListIDs []string
)
func init() { func init() {
filterModule = modules.Register("filter", filterPrep, filterStart, nil, "core", "intel") filterModule = modules.Register("filter", filterPrep, filterStart, nil, "core", "intel")
@ -48,18 +38,11 @@ func filterPrep() (err error) {
return err return err
} }
filterEnabled = config.Concurrent.GetAsBool(CfgOptionEnableFilterKey, true)
tunnelEnabled = config.Concurrent.GetAsBool(captain.CfgOptionEnableSPNKey, false)
return nil return nil
} }
func filterStart() error { func filterStart() error {
// TODO: Re-resolve IDs when filterlist index changes. getConfig()
resolvedIDs, err := filterlists.ResolveListIDs(unbreakFilterListIDs)
if err != nil {
log.Warningf("filter: failed to resolve unbreak filter list IDs: %s", err)
} else {
resolvedUnbreakFilterListIDs = resolvedIDs
}
return nil return nil
} }

13
firewall/interception.go
View file

@ -13,9 +13,12 @@ import (
"github.com/tevino/abool" "github.com/tevino/abool"
"golang.org/x/sync/singleflight" "golang.org/x/sync/singleflight"
"github.com/safing/portbase/api"
"github.com/safing/portbase/config"
"github.com/safing/portbase/log" "github.com/safing/portbase/log"
"github.com/safing/portbase/modules" "github.com/safing/portbase/modules"
"github.com/safing/portmaster/compat" "github.com/safing/portmaster/compat"
"github.com/safing/portmaster/core"
_ "github.com/safing/portmaster/core/base" _ "github.com/safing/portmaster/core/base"
"github.com/safing/portmaster/firewall/inspection" "github.com/safing/portmaster/firewall/inspection"
"github.com/safing/portmaster/firewall/interception" "github.com/safing/portmaster/firewall/interception"
@ -43,7 +46,14 @@ var (
ownPID = os.Getpid() ownPID = os.Getpid()
) )
// Config variables for interception module.
var (
devMode config.BoolOption
apiListenAddress config.StringOption
)
func init() { func init() {
// TODO: Move interception module to own package (dir).
interceptionModule = modules.Register("interception", interceptionPrep, interceptionStart, interceptionStop, "base", "updates", "network", "notifications") interceptionModule = modules.Register("interception", interceptionPrep, interceptionStart, interceptionStop, "base", "updates", "network", "notifications")
network.SetDefaultFirewallHandler(defaultHandler) network.SetDefaultFirewallHandler(defaultHandler)
@ -54,6 +64,9 @@ func interceptionPrep() error {
} }
func interceptionStart() error { func interceptionStart() error {
devMode = config.Concurrent.GetAsBool(core.CfgDevModeKey, false)
apiListenAddress = config.GetAsString(api.CfgDefaultListenAddressKey, "")
if err := registerMetrics(); err != nil { if err := registerMetrics(); err != nil {
return err return err
} }

14
firewall/interception/nfq/packet.go
View file

@ -71,15 +71,15 @@ func (pkt *packet) LoadPacketData() error {
} }
// TODO(ppacher): revisit the following behavior: // TODO(ppacher): revisit the following behavior:
// The legacy implementation of nfqueue (and the interception) module
// always accept a packet but may mark it so that a subsequent rule in
// the C17 chain drops, rejects or modifies it.
// //
// For drop/return we could use the actual nfQueue verdicts Drop and Stop. // The legacy implementation of nfqueue (and the interception) module
// Re-routing to local NS or SPN can be done by modifying the packet here // always accept a packet but may mark it so that a subsequent rule in
// and using SetVerdictModPacket and reject can be implemented using a simple // the C17 chain drops, rejects or modifies it.
// raw-socket.
// //
// For drop/return we could use the actual nfQueue verdicts Drop and Stop.
// Re-routing to local NS or SPN can be done by modifying the packet here
// and using SetVerdictModPacket and reject can be implemented using a simple
// raw-socket.
func (pkt *packet) mark(mark int) (err error) { func (pkt *packet) mark(mark int) (err error) {
if pkt.verdictPending.SetToIf(false, true) { if pkt.verdictPending.SetToIf(false, true) {
defer close(pkt.verdictSet) defer close(pkt.verdictSet)

2
firewall/master.go
View file

@ -14,6 +14,7 @@ import (
"github.com/safing/portbase/log" "github.com/safing/portbase/log"
"github.com/safing/portmaster/detection/dga" "github.com/safing/portmaster/detection/dga"
"github.com/safing/portmaster/intel/customlists" "github.com/safing/portmaster/intel/customlists"
"github.com/safing/portmaster/intel/filterlists"
"github.com/safing/portmaster/netenv" "github.com/safing/portmaster/netenv"
"github.com/safing/portmaster/network" "github.com/safing/portmaster/network"
"github.com/safing/portmaster/network/netutils" "github.com/safing/portmaster/network/netutils"
@ -435,6 +436,7 @@ func checkFilterLists(ctx context.Context, conn *network.Connection, p *profile.
switch result { switch result {
case endpoints.Denied: case endpoints.Denied:
// If the connection matches a filter list, check if the "unbreak" list matches too and abort blocking. // If the connection matches a filter list, check if the "unbreak" list matches too and abort blocking.
resolvedUnbreakFilterListIDs := filterlists.GetUnbreakFilterListIDs()
for _, blockedListID := range conn.Entity.BlockedByLists { for _, blockedListID := range conn.Entity.BlockedByLists {
for _, unbreakListID := range resolvedUnbreakFilterListIDs { for _, unbreakListID := range resolvedUnbreakFilterListIDs {
if blockedListID == unbreakListID { if blockedListID == unbreakListID {

5
firewall/tunnel.go
View file

@ -114,6 +114,11 @@ func checkTunneling(ctx context.Context, conn *network.Connection, pkt packet.Pa
RoutingProfile: layeredProfile.SPNRoutingAlgorithm(), RoutingProfile: layeredProfile.SPNRoutingAlgorithm(),
} }
// Add required verified owners if community nodes should not be used.
if !useCommunityNodes() {
conn.TunnelOpts.RequireVerifiedOwners = captain.NonCommunityVerifiedOwners
}
// If we have any exit hub policies, we need to raise the routing algorithm at least to single-hop. // If we have any exit hub policies, we need to raise the routing algorithm at least to single-hop.
if conn.TunnelOpts.RoutingProfile == navigator.RoutingProfileHomeID && if conn.TunnelOpts.RoutingProfile == navigator.RoutingProfileHomeID &&
conn.TunnelOpts.HubPoliciesAreSet() { conn.TunnelOpts.HubPoliciesAreSet() {

17
go.mod
View file

@ -1,10 +1,9 @@
module github.com/safing/portmaster module github.com/safing/portmaster
go 1.18 go 1.19
require ( require (
github.com/agext/levenshtein v1.2.3 github.com/agext/levenshtein v1.2.3
github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d
github.com/coreos/go-iptables v0.6.0 github.com/coreos/go-iptables v0.6.0
github.com/florianl/go-nfqueue v1.3.1 github.com/florianl/go-nfqueue v1.3.1
github.com/ghodss/yaml v1.0.0 github.com/ghodss/yaml v1.0.0
@ -16,16 +15,17 @@ require (
github.com/miekg/dns v1.1.50 github.com/miekg/dns v1.1.50
github.com/oschwald/maxminddb-golang v1.10.0 github.com/oschwald/maxminddb-golang v1.10.0
github.com/safing/portbase v0.15.1 github.com/safing/portbase v0.15.1
github.com/safing/spn v0.4.15 github.com/safing/spn v0.5.0
github.com/shirou/gopsutil v3.21.11+incompatible github.com/shirou/gopsutil v3.21.11+incompatible
github.com/spf13/cobra v1.5.0 github.com/spf13/cobra v1.5.0
github.com/spkg/zipfs v0.7.1
github.com/stretchr/testify v1.8.0 github.com/stretchr/testify v1.8.0
github.com/tannerryan/ring v1.1.2 github.com/tannerryan/ring v1.1.2
github.com/tevino/abool v1.2.0 github.com/tevino/abool v1.2.0
github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b
golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde
golang.org/x/sys v0.0.0-20220823224334-20c2bfdbfe24 golang.org/x/sys v0.0.0-20220829200755-d48e67d00261
zombiezen.com/go/sqlite v0.10.1 zombiezen.com/go/sqlite v0.10.1
) )
@ -47,7 +47,6 @@ require (
github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/josharian/native v1.0.0 // indirect github.com/josharian/native v1.0.0 // indirect
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mdlayher/netlink v1.6.0 // indirect github.com/mdlayher/netlink v1.6.0 // indirect
github.com/mdlayher/socket v0.2.3 // indirect github.com/mdlayher/socket v0.2.3 // indirect
@ -76,13 +75,13 @@ require (
github.com/x448/float16 v0.8.4 // indirect github.com/x448/float16 v0.8.4 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect go.etcd.io/bbolt v1.3.6 // indirect
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 // indirect golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect
golang.org/x/tools v0.1.12 // indirect golang.org/x/tools v0.1.12 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect
modernc.org/libc v1.17.0 // indirect modernc.org/libc v1.17.1 // indirect
modernc.org/mathutil v1.5.0 // indirect modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.2.0 // indirect modernc.org/memory v1.2.1 // indirect
modernc.org/sqlite v1.18.1 // indirect modernc.org/sqlite v1.18.1 // indirect
) )

27
go.sum
View file

@ -236,7 +236,6 @@ github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv
github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d h1:O+gcIbHv8EocDRI8swPGYI6XPJDbdZ66jeXqfoXifLE=
github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d/go.mod h1:Da90oEbCMTyeoWRBoWQHAmajIlLPjji2U2w7HJGAnuY= github.com/cookieo9/resources-go v0.0.0-20150225115733-d27c04069d0d/go.mod h1:Da90oEbCMTyeoWRBoWQHAmajIlLPjji2U2w7HJGAnuY=
github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@ -602,7 +601,6 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM=
@ -832,7 +830,6 @@ github.com/safing/portbase v0.14.2/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6X
github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= github.com/safing/portbase v0.14.3/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= github.com/safing/portbase v0.14.4/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI= github.com/safing/portbase v0.14.5/go.mod h1:z9sRR/vqohAGdYSSx2B+o8tND4WVvcxPL6XBBtN3bDI=
github.com/safing/portbase v0.15.0 h1:WwJpnERbNfzqDkEHt39l1TbBhnvf/oB2ZKj6fkaMSTM=
github.com/safing/portbase v0.15.0/go.mod h1:BinbSUlbOdsHTBSE8+WkKbR1bXNMlsbhhAW12EBxsUo= github.com/safing/portbase v0.15.0/go.mod h1:BinbSUlbOdsHTBSE8+WkKbR1bXNMlsbhhAW12EBxsUo=
github.com/safing/portbase v0.15.1 h1:s4AzyMSF26/b0CPmyHvKJSG9nW+u42+eIxlIKyp+J1U= github.com/safing/portbase v0.15.1 h1:s4AzyMSF26/b0CPmyHvKJSG9nW+u42+eIxlIKyp+J1U=
github.com/safing/portbase v0.15.1/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4fdlOMS91Y= github.com/safing/portbase v0.15.1/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4fdlOMS91Y=
@ -878,6 +875,8 @@ github.com/safing/spn v0.4.13/go.mod h1:rBeimIc1FHQOhX7lTh/LaFGRotmnwZIDWUSsPyeI
github.com/safing/spn v0.4.14/go.mod h1:GwGVU+GSmPQQXorKVDXIgieJZ16e70aBV5CcG0nAVRg= github.com/safing/spn v0.4.14/go.mod h1:GwGVU+GSmPQQXorKVDXIgieJZ16e70aBV5CcG0nAVRg=
github.com/safing/spn v0.4.15 h1:hiGAQgkfxm3yfakkt7mVmkUfYxtE0pgODH9LyloEr/o= github.com/safing/spn v0.4.15 h1:hiGAQgkfxm3yfakkt7mVmkUfYxtE0pgODH9LyloEr/o=
github.com/safing/spn v0.4.15/go.mod h1:mgVGd4sUdem78irl/pSx/JqOM/Ow85pJqKKqPpj1Ikk= github.com/safing/spn v0.4.15/go.mod h1:mgVGd4sUdem78irl/pSx/JqOM/Ow85pJqKKqPpj1Ikk=
github.com/safing/spn v0.5.0 h1:gOkcz1io7/4A4rZislqlm3w+gzVqO5861tbbib0EBUM=
github.com/safing/spn v0.5.0/go.mod h1:BGzLv966GfYZXc4EUFk04QfrfkpzDdaNObkKTRPbdpk=
github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
@ -935,6 +934,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM
github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg= github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM=
github.com/spkg/zipfs v0.7.1 h1:+2X5lvNHTybnDMQZAIHgedRXZK1WXdc+94R/P5v2XWE=
github.com/spkg/zipfs v0.7.1/go.mod h1:48LW+/Rh1G7aAav1ew1PdlYn52T+LM+ARmSHfDNJvg8=
github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -1120,8 +1121,8 @@ golang.org/x/crypto v0.0.0-20220507011949-2cf3adece122/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220513210258-46612604a0f9/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8 h1:GIAS/yBem/gq2MUqgNIzUHW7cJMmx3TGZOrnyYaNQ6c= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 h1:Y/gsMcFOcR+6S6f3YeMKl5g+dZMEWqcz5Czj/GWYbkM=
golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@ -1246,8 +1247,8 @@ golang.org/x/net v0.0.0-20220513224357-95641704303c/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220728211354-c7608f3a8462/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c h1:JVAXQ10yGGVbSyoer5VILysz6YKjdNT2bsvlayjqhes= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b h1:ZmngSVLe/wycRns9MKikG9OWIEjGcGAkacif7oYQaUY=
golang.org/x/net v0.0.0-20220822230855-b0a4917ee28c/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@ -1415,8 +1416,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220731174439-a90be440212d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220823224334-20c2bfdbfe24 h1:TyKJRhyo17yWxOMCTHKWrc5rddHORMlnZ/j57umaUd8= golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 h1:v6hYoSR9T5oet+pMXwUWkbiVqx/63mlHjefrHmxwfeY=
golang.org/x/sys v0.0.0-20220823224334-20c2bfdbfe24/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@ -1750,15 +1751,15 @@ modernc.org/libc v1.16.1/go.mod h1:JjJE0eu4yeK7tab2n4S1w8tlWd9MxXLRzheaRnAKymU=
modernc.org/libc v1.16.7/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= modernc.org/libc v1.16.7/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU= modernc.org/libc v1.16.17/go.mod h1:hYIV5VZczAmGZAnG15Vdngn5HSF5cSkbvfz2B7GRuVU=
modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA= modernc.org/libc v1.16.19/go.mod h1:p7Mg4+koNjc8jkqwcoFBJx7tXkpj00G77X7A72jXPXA=
modernc.org/libc v1.17.0 h1:nbL2Lv0I323wLc1GmTh/AqVtI9JeBVc7Nhapdg9EONs= modernc.org/libc v1.17.1 h1:Q8/Cpi36V/QBfuQaFVeisEBs3WqoGAJprZzmf7TfEYI=
modernc.org/libc v1.17.0/go.mod h1:XsgLldpP4aWlPlsjqKRdHPqCxCjISdHfM/yeWC5GyW0= modernc.org/libc v1.17.1/go.mod h1:FZ23b+8LjxZs7XtFMbSzL/EhPxNbfZbErxEHc7cbD9s=
modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.2.2/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.4.1/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ=
modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E=
modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/memory v1.1.1/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw=
modernc.org/memory v1.2.0 h1:zXehBrt9n+Pjn+4RoRCZ0KqRA/0ePFqcecxZ/hXCIVw= modernc.org/memory v1.2.1 h1:dkRh86wgmq/bJu2cAS2oqBCz/KsMZU7TUM4CibQ7eBs=
modernc.org/memory v1.2.0/go.mod h1:/0wo5ibyrQiaoUoH7f9D8dnglAmILJ5/cxZlRECf+Nw= modernc.org/memory v1.2.1/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU=
modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= modernc.org/opt v0.1.1/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sqlite v1.17.3/go.mod h1:10hPVYar9C0kfXuTWGz8s0XtB8uAGymUy51ZzStYe3k= modernc.org/sqlite v1.17.3/go.mod h1:10hPVYar9C0kfXuTWGz8s0XtB8uAGymUy51ZzStYe3k=
modernc.org/sqlite v1.18.0/go.mod h1:B9fRWZacNxJBHoCJZQr1R54zhVn3fjfl0aszflrTSxY= modernc.org/sqlite v1.18.0/go.mod h1:B9fRWZacNxJBHoCJZQr1R54zhVn3fjfl0aszflrTSxY=

56
intel/entity.go
View file

@ -237,8 +237,8 @@ func (e *Entity) GetIP() (net.IP, bool) {
func (e *Entity) getLocation(ctx context.Context) { func (e *Entity) getLocation(ctx context.Context) {
e.fetchLocationOnce.Do(func() { e.fetchLocationOnce.Do(func() {
// need IP! // Only check if we have a global IP address.
if e.IP == nil { if e.IP == nil || !e.IPScope.IsGlobal() {
return return
} }
@ -254,6 +254,31 @@ func (e *Entity) getLocation(ctx context.Context) {
e.Coordinates = &loc.Coordinates e.Coordinates = &loc.Coordinates
e.ASN = loc.AutonomousSystemNumber e.ASN = loc.AutonomousSystemNumber
e.ASOrg = loc.AutonomousSystemOrganization e.ASOrg = loc.AutonomousSystemOrganization
// Log result.
if log.GetLogLevel() == log.TraceLevel {
// Build flags
var flags string
if loc.IsAnycast {
flags += " anycast"
}
if loc.IsSatelliteProvider {
flags += " satellite"
}
if loc.IsAnonymousProxy {
flags += " anonymous"
}
// Log location
log.Tracer(ctx).Tracef(
"intel: located %s in %s (AS%d by %s)%s",
e.IP,
loc.Country.ISOCode,
loc.AutonomousSystemNumber,
loc.AutonomousSystemOrganization,
flags,
)
}
}) })
} }
@ -318,7 +343,6 @@ func (e *Entity) getDomainLists(ctx context.Context) {
return return
} }
log.Tracer(ctx).Tracef("intel: loading domain list for %s", domain)
e.loadDomainListOnce.Do(func() { e.loadDomainListOnce.Do(func() {
domainsToInspect := []string{domain} domainsToInspect := []string{domain}
@ -347,7 +371,10 @@ func (e *Entity) getDomainLists(ctx context.Context) {
return return
} }
e.mergeList(d, list) if len(list) > 0 {
log.Tracer(ctx).Tracef("intel: loaded domain lists for %s: %s", d, strings.Join(list, ", "))
e.mergeList(d, list)
}
} }
e.domainListLoaded = true e.domainListLoaded = true
}) })
@ -389,7 +416,6 @@ func (e *Entity) getASNLists(ctx context.Context) {
return return
} }
log.Tracer(ctx).Tracef("intel: loading ASN list for %d", asn)
e.loadAsnListOnce.Do(func() { e.loadAsnListOnce.Do(func() {
asnStr := fmt.Sprintf("%d", asn) asnStr := fmt.Sprintf("%d", asn)
list, err := filterlists.LookupASNString(asnStr) list, err := filterlists.LookupASNString(asnStr)
@ -399,8 +425,12 @@ func (e *Entity) getASNLists(ctx context.Context) {
return return
} }
if len(list) > 0 {
log.Tracer(ctx).Tracef("intel: loaded ASN lists for %s: %s", asnStr, strings.Join(list, ", "))
e.mergeList(asnStr, list)
}
e.asnListLoaded = true e.asnListLoaded = true
e.mergeList(asnStr, list)
}) })
} }
@ -414,7 +444,6 @@ func (e *Entity) getCountryLists(ctx context.Context) {
return return
} }
log.Tracer(ctx).Tracef("intel: loading country list for %s", country)
e.loadCountryListOnce.Do(func() { e.loadCountryListOnce.Do(func() {
list, err := filterlists.LookupCountry(country) list, err := filterlists.LookupCountry(country)
if err != nil { if err != nil {
@ -423,8 +452,12 @@ func (e *Entity) getCountryLists(ctx context.Context) {
return return
} }
if len(list) > 0 {
log.Tracer(ctx).Tracef("intel: loaded country lists for %s: %s", country, strings.Join(list, ", "))
e.mergeList(country, list)
}
e.countryListLoaded = true e.countryListLoaded = true
e.mergeList(country, list)
}) })
} }
@ -443,7 +476,6 @@ func (e *Entity) getIPLists(ctx context.Context) {
return return
} }
log.Tracer(ctx).Tracef("intel: loading IP list for %s", ip)
e.loadIPListOnce.Do(func() { e.loadIPListOnce.Do(func() {
list, err := filterlists.LookupIP(ip) list, err := filterlists.LookupIP(ip)
if err != nil { if err != nil {
@ -452,8 +484,12 @@ func (e *Entity) getIPLists(ctx context.Context) {
return return
} }
if len(list) > 0 {
log.Tracer(ctx).Tracef("intel: loaded IP lists for %s: %s", ip.String(), strings.Join(list, ", "))
e.mergeList(ip.String(), list)
}
e.ipListLoaded = true e.ipListLoaded = true
e.mergeList(ip.String(), list)
}) })
} }

36
intel/filterlists/index.go
View file

@ -196,8 +196,12 @@ func updateListIndex() error {
listIndexUpdate.Version(), listIndexUpdate.Version(),
) )
default: default:
log.Debug("filterlists: index is up to date")
// List is in cache and current, there is nothing to do. // List is in cache and current, there is nothing to do.
log.Debug("filterlists: index is up to date")
// Update the unbreak filter list IDs on initial load.
updateUnbreakFilterListIDs()
return nil return nil
} }
case listIndexUpdate.UpgradeAvailable(): case listIndexUpdate.UpgradeAvailable():
@ -225,6 +229,9 @@ func updateListIndex() error {
} }
log.Debugf("intel/filterlists: updated list index in cache to %s", index.Version) log.Debugf("intel/filterlists: updated list index in cache to %s", index.Version)
// Update the unbreak filter list IDs after an update.
updateUnbreakFilterListIDs()
return nil return nil
} }
@ -252,3 +259,30 @@ func ResolveListIDs(ids []string) ([]string, error) {
return resolved, nil return resolved, nil
} }
var (
unbreakCategoryIDs = []string{"UNBREAK"}
unbreakIDs []string
unbreakIDsLock sync.Mutex
)
// GetUnbreakFilterListIDs returns the resolved list of all unbreak filter lists.
func GetUnbreakFilterListIDs() []string {
unbreakIDsLock.Lock()
defer unbreakIDsLock.Unlock()
return unbreakIDs
}
func updateUnbreakFilterListIDs() {
unbreakIDsLock.Lock()
defer unbreakIDsLock.Unlock()
resolvedIDs, err := ResolveListIDs(unbreakCategoryIDs)
if err != nil {
log.Warningf("filter: failed to resolve unbreak filter list IDs: %s", err)
} else {
unbreakIDs = resolvedIDs
}
}

3
intel/geoip/location.go
View file

@ -26,6 +26,9 @@ type Location struct {
Coordinates Coordinates `maxminddb:"location"` Coordinates Coordinates `maxminddb:"location"`
AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"` AutonomousSystemNumber uint `maxminddb:"autonomous_system_number"`
AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"` AutonomousSystemOrganization string `maxminddb:"autonomous_system_organization"`
IsAnycast bool `maxminddb:"is_anycast"`
IsSatelliteProvider bool `maxminddb:"is_satellite_provider"`
IsAnonymousProxy bool `maxminddb:"is_anonymous_proxy"`
} }
// Coordinates holds geographic coordinates and their estimated accuracy. // Coordinates holds geographic coordinates and their estimated accuracy.

4
netenv/online-status.go
View file

@ -468,7 +468,7 @@ func checkOnlineStatus(ctx context.Context) {
} }
request := (&http.Request{ request := (&http.Request{
Method: "GET", Method: http.MethodGet,
URL: parsedPortalTestURL, URL: parsedPortalTestURL,
Close: true, Close: true,
}).WithContext(ctx) }).WithContext(ctx)
@ -495,7 +495,7 @@ func checkOnlineStatus(ctx context.Context) {
} }
// direct response // direct response
if response.StatusCode == 200 { if response.StatusCode == http.StatusOK {
updateOnlineStatus(StatusPortal, &url.URL{ updateOnlineStatus(StatusPortal, &url.URL{
Scheme: "http", Scheme: "http",
Host: SpecialCaptivePortalDomain, Host: SpecialCaptivePortalDomain,

5
netquery/database.go
View file

@ -10,10 +10,10 @@ import (
"sync" "sync"
"time" "time"
"github.com/jackc/puddle/v2"
"zombiezen.com/go/sqlite" "zombiezen.com/go/sqlite"
"zombiezen.com/go/sqlite/sqlitex" "zombiezen.com/go/sqlite/sqlitex"
"github.com/jackc/puddle/v2"
"github.com/safing/portbase/log" "github.com/safing/portbase/log"
"github.com/safing/portmaster/netquery/orm" "github.com/safing/portmaster/netquery/orm"
"github.com/safing/portmaster/network" "github.com/safing/portmaster/network"
@ -108,7 +108,6 @@ type (
// (see Execute). To perform database writes use either Save() or ExecuteWrite(). // (see Execute). To perform database writes use either Save() or ExecuteWrite().
// Note that write connections are serialized by the Database object before being // Note that write connections are serialized by the Database object before being
// handed over to SQLite. // handed over to SQLite.
//
func New(path string) (*Database, error) { func New(path string) (*Database, error) {
constructor := func(ctx context.Context) (*sqlite.Conn, error) { constructor := func(ctx context.Context) (*sqlite.Conn, error) {
c, err := sqlite.OpenConn( c, err := sqlite.OpenConn(
@ -331,7 +330,7 @@ func (db *Database) Save(ctx context.Context, conn Conn) error {
for key := range connMap { for key := range connMap {
keys = append(keys, key) keys = append(keys, key)
} }
sort.Sort(sort.StringSlice(keys)) sort.Strings(keys)
for _, key := range keys { for _, key := range keys {
value := connMap[key] value := connMap[key]

1
netquery/orm/decoder.go
View file

@ -63,7 +63,6 @@ type (
// Decoding hooks configured in cfg are executed before trying to decode basic types and may // Decoding hooks configured in cfg are executed before trying to decode basic types and may
// be specified to provide support for special types. // be specified to provide support for special types.
// See DatetimeDecoder() for an example of a DecodeHook that handles graceful time.Time conversion. // See DatetimeDecoder() for an example of a DecodeHook that handles graceful time.Time conversion.
//
func DecodeStmt(ctx context.Context, schema *TableSchema, stmt Stmt, result interface{}, cfg DecodeConfig) error { func DecodeStmt(ctx context.Context, schema *TableSchema, stmt Stmt, result interface{}, cfg DecodeConfig) error {
// make sure we got something to decode into ... // make sure we got something to decode into ...
if result == nil { if result == nil {

11
netquery/orm/query_runner.go
View file

@ -94,13 +94,12 @@ func WithDecodeConfig(cfg DecodeConfig) QueryOption {
// //
// Example: // Example:
// //
// var result []struct{ // var result []struct{
// Count int `sqlite:"rowCount"` // Count int `sqlite:"rowCount"`
// } // }
//
// err := RunQuery(ctx, conn, "SELECT COUNT(*) AS rowCount FROM table", WithResult(&result))
// fmt.Println(result[0].Count)
// //
// err := RunQuery(ctx, conn, "SELECT COUNT(*) AS rowCount FROM table", WithResult(&result))
// fmt.Println(result[0].Count)
func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...QueryOption) error { func RunQuery(ctx context.Context, conn *sqlite.Conn, sql string, modifiers ...QueryOption) error {
args := queryOpts{ args := queryOpts{
DecodeConfig: DefaultDecodeConfig, DecodeConfig: DefaultDecodeConfig,

1
netquery/query.go
View file

@ -17,6 +17,7 @@ import (
// Collection of Query and Matcher types. // Collection of Query and Matcher types.
// NOTE: whenever adding support for new operators make sure // NOTE: whenever adding support for new operators make sure
// to update UnmarshalJSON as well. // to update UnmarshalJSON as well.
//
//nolint:golint //nolint:golint
type ( type (
Query map[string][]Matcher Query map[string][]Matcher

12
network/packet/packet.go
View file

@ -125,10 +125,12 @@ func (pkt *Base) createConnectionID() {
// MatchesAddress checks if a the packet matches a given endpoint (remote or local) in protocol, network and port. // MatchesAddress checks if a the packet matches a given endpoint (remote or local) in protocol, network and port.
// //
// Comparison matrix: // Comparison matrix:
// IN OUT //
// ====== IN OUT
//
// Local Dst Src // Local Dst Src
// Remote Src Dst // Remote Src Dst
//. // .
func (pkt *Base) MatchesAddress(remote bool, protocol IPProtocol, network *net.IPNet, port uint16) bool { func (pkt *Base) MatchesAddress(remote bool, protocol IPProtocol, network *net.IPNet, port uint16) bool {
if pkt.info.Protocol != protocol { if pkt.info.Protocol != protocol {
return false return false
@ -154,10 +156,12 @@ func (pkt *Base) MatchesAddress(remote bool, protocol IPProtocol, network *net.I
// MatchesIP checks if a the packet matches a given endpoint (remote or local) IP. // MatchesIP checks if a the packet matches a given endpoint (remote or local) IP.
// //
// Comparison matrix: // Comparison matrix:
// IN OUT //
// ====== IN OUT
//
// Local Dst Src // Local Dst Src
// Remote Src Dst // Remote Src Dst
//. // .
func (pkt *Base) MatchesIP(endpoint bool, network *net.IPNet) bool { func (pkt *Base) MatchesIP(endpoint bool, network *net.IPNet) bool {
if pkt.info.Inbound != endpoint { if pkt.info.Inbound != endpoint {
if network.Contains(pkt.info.Src) { if network.Contains(pkt.info.Src) {

8
process/doc.go
View file

@ -1,7 +1,3 @@
/* // Package process fetches process and socket information from the operating system.
// It can find the process owning a network connection.
Package process fetches process and socket information from the operating system.
It can find the process owning a network connection.
*/
package process package process

13
resolver/doc.go
View file

@ -1,22 +1,22 @@
/* /*
Package resolver is responsible for querying DNS. Package resolver is responsible for querying DNS.
DNS Servers # DNS Servers
Internal lists of resolvers to use are built on start and rebuilt on every config or network change. Internal lists of resolvers to use are built on start and rebuilt on every config or network change.
Configured DNS servers are prioritized over servers assigned by dhcp. Domain and search options (here referred to as "search scopes") are being considered. Configured DNS servers are prioritized over servers assigned by dhcp. Domain and search options (here referred to as "search scopes") are being considered.
Security # Security
Usage of DNS Servers can be regulated using the configuration: Usage of DNS Servers can be regulated using the configuration:
DoNotUseAssignedDNS // Do not use DNS servers assigned by DHCP DoNotUseAssignedDNS // Do not use DNS servers assigned by DHCP
DoNotUseMDNS // Do not use mDNS DoNotUseMDNS // Do not use mDNS
DoNotForwardSpecialDomains // Do not forward special domains to local resolvers, except if they have a search scope for it DoNotForwardSpecialDomains // Do not forward special domains to local resolvers, except if they have a search scope for it
Note: The DHCP options "domain" and "search" are ignored for servers assigned by DHCP that do not reside within local address space. Note: The DHCP options "domain" and "search" are ignored for servers assigned by DHCP that do not reside within local address space.
Resolving DNS # Resolving DNS
Various different queries require the resolver to behave in different manner: Various different queries require the resolver to behave in different manner:
@ -24,6 +24,5 @@ Queries for "localhost." are immediately responded with 127.0.0.1 and ::1, for A
Reverse lookups on local address ranges (10/8, 172.16/12, 192.168/16, fe80::/7) will be tried against every local resolver and finally mDNS until a successful, non-NXDomain answer is received. Reverse lookups on local address ranges (10/8, 172.16/12, 192.168/16, fe80::/7) will be tried against every local resolver and finally mDNS until a successful, non-NXDomain answer is received.
Special domains ("example.", "example.com.", "example.net.", "example.org.", "invalid.", "test.", "onion.") are resolved using search scopes and local resolvers. Special domains ("example.", "example.com.", "example.net.", "example.org.", "invalid.", "test.", "onion.") are resolved using search scopes and local resolvers.
All other domains are resolved using search scopes and all available resolvers. All other domains are resolved using search scopes and all available resolvers.
*/ */
package resolver package resolver

4
resolver/resolver-https.go
View file

@ -90,7 +90,9 @@ func (hr *HTTPSResolver) Query(ctx context.Context, q *Query) (*RRCache, error)
if err != nil { if err != nil {
return nil, err return nil, err
} }
defer resp.Body.Close() defer func() {
_ = resp.Body.Close()
}()
// Try to read the result // Try to read the result
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)

6
resolver/resolver.go
View file

@ -30,7 +30,7 @@ const (
ServerSourceEnv = "env" ServerSourceEnv = "env"
) )
// DNS Resolver alias // DNS resolver scheme aliases.
const ( const (
HTTPSProtocol = "https" HTTPSProtocol = "https"
TLSProtocol = "tls" TLSProtocol = "tls"
@ -117,14 +117,14 @@ func (info *ResolverInfo) ID() string {
case ServerTypeEnv: case ServerTypeEnv:
info.id = ServerTypeEnv info.id = ServerTypeEnv
case ServerTypeDoH: case ServerTypeDoH:
info.id = fmt.Sprintf( info.id = fmt.Sprintf( //nolint:nosprintfhostport // Not used as URL.
"https://%s:%d#%s", "https://%s:%d#%s",
info.Domain, info.Domain,
info.Port, info.Port,
info.Source, info.Source,
) )
case ServerTypeDoT: case ServerTypeDoT:
info.id = fmt.Sprintf( info.id = fmt.Sprintf( //nolint:nosprintfhostport // Not used as URL.
"dot://%s:%d#%s", "dot://%s:%d#%s",
info.Domain, info.Domain,
info.Port, info.Port,

2
resolver/resolvers.go
View file

@ -10,9 +10,9 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/miekg/dns"
"golang.org/x/net/publicsuffix" "golang.org/x/net/publicsuffix"
"github.com/miekg/dns"
"github.com/safing/portbase/log" "github.com/safing/portbase/log"
"github.com/safing/portbase/utils" "github.com/safing/portbase/utils"
"github.com/safing/portmaster/netenv" "github.com/safing/portmaster/netenv"

3
resolver/rrcache.go
View file

@ -16,7 +16,8 @@ import (
// RRCache is a single-use structure to hold a DNS response. // RRCache is a single-use structure to hold a DNS response.
// Persistence is handled through NameRecords because of a limitation of the // Persistence is handled through NameRecords because of a limitation of the
// underlying dns library. // underlying dns library.
//nolint:maligned // TODO //
//nolint:maligned
type RRCache struct { type RRCache struct {
// Respnse Header // Respnse Header
Domain string Domain string

13
status/threat.go
View file

@ -37,14 +37,13 @@ type ThreatPayload struct {
// //
// Example: // Example:
// //
// threat := NewThreat("portscan", "Someone is scanning you"). // threat := NewThreat("portscan", "Someone is scanning you").
// SetData(portscanResult). // SetData(portscanResult).
// SetMitigationLevel(SecurityLevelExtreme). // SetMitigationLevel(SecurityLevelExtreme).
// Publish() // Publish()
//
// // Once you're done, delete the threat
// threat.Delete().Publish()
// //
// Once you're done, delete the threat
// threat.Delete().Publish()
func NewThreat(id, title, msg string) *Threat { func NewThreat(id, title, msg string) *Threat {
t := &Threat{ t := &Threat{
Notification: &notifications.Notification{ Notification: &notifications.Notification{

16
ui/api.go
View file

@ -1,8 +1,6 @@
package ui package ui
import ( import (
resources "github.com/cookieo9/resources-go"
"github.com/safing/portbase/api" "github.com/safing/portbase/api"
"github.com/safing/portbase/log" "github.com/safing/portbase/log"
) )
@ -22,16 +20,18 @@ func reloadUI(_ *api.Request) (msg string, err error) {
appsLock.Lock() appsLock.Lock()
defer appsLock.Unlock() defer appsLock.Unlock()
// close all bundles. // Close all archives.
for id, bundle := range apps { for id, archiveFS := range apps {
err := bundle.Close() err := archiveFS.Close()
if err != nil { if err != nil {
log.Warningf("ui: failed to close bundle %s: %s", id, err) log.Warningf("ui: failed to close archive %s: %s", id, err)
} }
} }
// Reset index. // Reset index.
apps = make(map[string]*resources.BundleSequence) for key := range apps {
delete(apps, key)
}
return "all ui bundles successfully reloaded", nil return "all ui archives successfully reloaded", nil
} }

52
ui/serve.go
View file

@ -6,11 +6,12 @@ import (
"io" "io"
"net/http" "net/http"
"net/url" "net/url"
"os"
"path/filepath" "path/filepath"
"strings" "strings"
"sync" "sync"
resources "github.com/cookieo9/resources-go" "github.com/spkg/zipfs"
"github.com/safing/portbase/api" "github.com/safing/portbase/api"
"github.com/safing/portbase/log" "github.com/safing/portbase/log"
@ -20,7 +21,7 @@ import (
) )
var ( var (
apps = make(map[string]*resources.BundleSequence) apps = make(map[string]*zipfs.FileSystem)
appsLock sync.RWMutex appsLock sync.RWMutex
) )
@ -28,7 +29,7 @@ func registerRoutes() error {
// Server assets. // Server assets.
api.RegisterHandler( api.RegisterHandler(
"/assets/{resPath:[a-zA-Z0-9/\\._-]+}", "/assets/{resPath:[a-zA-Z0-9/\\._-]+}",
&bundleServer{defaultModuleName: "assets"}, &archiveServer{defaultModuleName: "assets"},
) )
// Add slash to plain module namespaces. // Add slash to plain module namespaces.
@ -38,7 +39,7 @@ func registerRoutes() error {
) )
// Serve modules. // Serve modules.
srv := &bundleServer{} srv := &archiveServer{}
api.RegisterHandler("/ui/modules/{moduleName:[a-z]+}/", srv) api.RegisterHandler("/ui/modules/{moduleName:[a-z]+}/", srv)
api.RegisterHandler("/ui/modules/{moduleName:[a-z]+}/{resPath:[a-zA-Z0-9/\\._-]+}", srv) api.RegisterHandler("/ui/modules/{moduleName:[a-z]+}/{resPath:[a-zA-Z0-9/\\._-]+}", srv)
@ -51,17 +52,17 @@ func registerRoutes() error {
return nil return nil
} }
type bundleServer struct { type archiveServer struct {
defaultModuleName string defaultModuleName string
} }
func (bs *bundleServer) BelongsTo() *modules.Module { return module } func (bs *archiveServer) BelongsTo() *modules.Module { return module }
func (bs *bundleServer) ReadPermission(*http.Request) api.Permission { return api.PermitAnyone } func (bs *archiveServer) ReadPermission(*http.Request) api.Permission { return api.PermitAnyone }
func (bs *bundleServer) WritePermission(*http.Request) api.Permission { return api.NotSupported } func (bs *archiveServer) WritePermission(*http.Request) api.Permission { return api.NotSupported }
func (bs *bundleServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (bs *archiveServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Get request context. // Get request context.
ar := api.GetAPIRequest(r) ar := api.GetAPIRequest(r)
if ar == nil { if ar == nil {
@ -85,10 +86,10 @@ func (bs *bundleServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
} }
appsLock.RLock() appsLock.RLock()
bundle, ok := apps[moduleName] archiveFS, ok := apps[moduleName]
appsLock.RUnlock() appsLock.RUnlock()
if ok { if ok {
ServeFileFromBundle(w, r, moduleName, bundle, resPath) ServeFileFromArchive(w, r, moduleName, archiveFS, resPath)
return return
} }
@ -105,39 +106,38 @@ func (bs *bundleServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
return return
} }
// open bundle // Open archive from disk.
newBundle, err := resources.OpenZip(zipFile.Path()) archiveFS, err = zipfs.New(zipFile.Path())
if err != nil { if err != nil {
log.Tracef("ui: error prepping module %s: %s", moduleName, err) log.Tracef("ui: error prepping module %s: %s", moduleName, err)
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
} }
bundle = &resources.BundleSequence{newBundle}
appsLock.Lock() appsLock.Lock()
apps[moduleName] = bundle apps[moduleName] = archiveFS
appsLock.Unlock() appsLock.Unlock()
ServeFileFromBundle(w, r, moduleName, bundle, resPath) ServeFileFromArchive(w, r, moduleName, archiveFS, resPath)
} }
// ServeFileFromBundle serves a file from the given bundle. // ServeFileFromArchive serves a file from the given archive.
func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundleName string, bundle *resources.BundleSequence, path string) { func ServeFileFromArchive(w http.ResponseWriter, r *http.Request, archiveName string, archiveFS *zipfs.FileSystem, path string) {
readCloser, err := bundle.Open(path) readCloser, err := archiveFS.Open(path)
if err != nil { if err != nil {
if errors.Is(err, resources.ErrNotFound) { if os.IsNotExist(err) {
// Check if there is a base index.html file we can serve instead. // Check if there is a base index.html file we can serve instead.
var indexErr error var indexErr error
path = "index.html" path = "index.html"
readCloser, indexErr = bundle.Open(path) readCloser, indexErr = archiveFS.Open(path)
if indexErr != nil { if indexErr != nil {
// If we cannot get an index, continue with handling the original error. // If we cannot get an index, continue with handling the original error.
log.Tracef("ui: requested resource \"%s\" not found in bundle %s: %s", path, bundleName, err) log.Tracef("ui: requested resource \"%s\" not found in archive %s: %s", path, archiveName, err)
http.Error(w, err.Error(), http.StatusNotFound) http.Error(w, err.Error(), http.StatusNotFound)
return return
} }
} else { } else {
log.Tracef("ui: error opening module %s: %s", bundleName, err) log.Tracef("ui: error opening module %s: %s", archiveName, err)
http.Error(w, err.Error(), http.StatusInternalServerError) http.Error(w, err.Error(), http.StatusInternalServerError)
return return
} }
@ -152,12 +152,8 @@ func ServeFileFromBundle(w http.ResponseWriter, r *http.Request, bundleName stri
} }
} }
// TODO: Set content security policy
// For some reason, this breaks the ui client
// w.Header().Set("Content-Security-Policy", "default-src 'self'")
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
if r.Method != "HEAD" { if r.Method != http.MethodHead {
_, err = io.Copy(w, readCloser) _, err = io.Copy(w, readCloser)
if err != nil { if err != nil {
log.Errorf("ui: failed to serve file: %s", err) log.Errorf("ui: failed to serve file: %s", err)

14
updates/main.go
View file

@ -5,6 +5,7 @@ import (
"flag" "flag"
"fmt" "fmt"
"runtime" "runtime"
"sync/atomic"
"time" "time"
"github.com/safing/portbase/database" "github.com/safing/portbase/database"
@ -209,6 +210,8 @@ func DisableUpdateSchedule() error {
return nil return nil
} }
var updateFailedCnt = new(atomic.Int32)
func checkForUpdates(ctx context.Context) (err error) { func checkForUpdates(ctx context.Context) (err error) {
if !forceUpdate.SetToIf(true, false) && !enableUpdates() { if !forceUpdate.SetToIf(true, false) && !enableUpdates() {
log.Warningf("updates: automatic updates are disabled") log.Warningf("updates: automatic updates are disabled")
@ -216,7 +219,9 @@ func checkForUpdates(ctx context.Context) (err error) {
} }
defer func() { defer func() {
// Resolve any error and and send succes notification.
if err == nil { if err == nil {
updateFailedCnt.Store(0)
log.Infof("updates: successfully checked for updates") log.Infof("updates: successfully checked for updates")
module.Resolve(updateFailed) module.Resolve(updateFailed)
notifications.Notify(&notifications.Notification{ notifications.Notify(&notifications.Notification{
@ -232,7 +237,14 @@ func checkForUpdates(ctx context.Context) (err error) {
}, },
}, },
}) })
} else { return
}
// Log error in any case.
log.Errorf("updates: check failed: %s", err)
// Do not alert user if update failed for only a few times.
if updateFailedCnt.Add(1) > 3 {
notifications.NotifyWarn( notifications.NotifyWarn(
updateFailed, updateFailed,
"Update Check Failed", "Update Check Failed",

3
updates/upgrader.go
View file

@ -35,8 +35,7 @@ var (
pmCtrlUpdate *updater.File pmCtrlUpdate *updater.File
pmCoreUpdate *updater.File pmCoreUpdate *updater.File
spnHubUpdate *updater.File spnHubUpdate *updater.File
hubUpgradeStarted bool
rawVersionRegex = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+b?\*?$`) rawVersionRegex = regexp.MustCompile(`^[0-9]+\.[0-9]+\.[0-9]+b?\*?$`)
) )