diff --git a/api/endpoints.go b/api/endpoints.go index 265df69..37a8b45 100644 --- a/api/endpoints.go +++ b/api/endpoints.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "sort" "strconv" @@ -501,7 +500,7 @@ func readBody(w http.ResponseWriter, r *http.Request) (inputData []byte, ok bool } // Read and close body. - inputData, err := ioutil.ReadAll(r.Body) + inputData, err := io.ReadAll(r.Body) if err != nil { http.Error(w, "failed to read body"+err.Error(), http.StatusInternalServerError) return nil, false diff --git a/api/main_test.go b/api/main_test.go index 078f4b6..4bbc5a0 100644 --- a/api/main_test.go +++ b/api/main_test.go @@ -2,7 +2,6 @@ package api import ( "fmt" - "io/ioutil" "os" "testing" @@ -21,7 +20,7 @@ func TestMain(m *testing.M) { module.Enable() // tmp dir for data root (db & config) - tmpDir, err := ioutil.TempDir("", "portbase-testing-") + tmpDir, err := os.MkdirTemp("", "portbase-testing-") if err != nil { fmt.Fprintf(os.Stderr, "failed to create tmp dir: %s\n", err) os.Exit(1) diff --git a/config/persistence.go b/config/persistence.go index 0b5d40e..c88c83c 100644 --- a/config/persistence.go +++ b/config/persistence.go @@ -3,7 +3,7 @@ package config import ( "encoding/json" "fmt" - "io/ioutil" + "os" "path" "strings" "sync" @@ -34,7 +34,7 @@ func loadConfig(requireValidConfig bool) error { } // read config file - data, err := ioutil.ReadFile(configFilePath) + data, err := os.ReadFile(configFilePath) if err != nil { return err } @@ -93,7 +93,7 @@ func saveConfig() error { } // write file - return ioutil.WriteFile(configFilePath, data, 0o0600) + return os.WriteFile(configFilePath, data, 0o0600) } // JSONToMap parses and flattens a hierarchical json object. diff --git a/container/doc.go b/container/doc.go index 16fd161..76cc73c 100644 --- a/container/doc.go +++ b/container/doc.go @@ -5,23 +5,22 @@ // Byte slices added to the Container are not changed or appended, to not corrupt any other data that may be before and after the given slice. // If interested, consider the following example to understand why this is important: // -// package main +// package main // -// import ( -// "fmt" -// ) +// import ( +// "fmt" +// ) // -// func main() { -// a := []byte{0, 1,2,3,4,5,6,7,8,9} -// fmt.Printf("a: %+v\n", a) -// fmt.Printf("\nmaking changes...\n(we are not changing a directly)\n\n") -// b := a[2:6] -// c := append(b, 10, 11) -// fmt.Printf("b: %+v\n", b) -// fmt.Printf("c: %+v\n", c) -// fmt.Printf("a: %+v\n", a) -// } +// func main() { +// a := []byte{0, 1,2,3,4,5,6,7,8,9} +// fmt.Printf("a: %+v\n", a) +// fmt.Printf("\nmaking changes...\n(we are not changing a directly)\n\n") +// b := a[2:6] +// c := append(b, 10, 11) +// fmt.Printf("b: %+v\n", b) +// fmt.Printf("c: %+v\n", c) +// fmt.Printf("a: %+v\n", a) +// } // // run it here: https://play.golang.org/p/xu1BXT3QYeE -// package container diff --git a/database/database_test.go b/database/database_test.go index 33cb2df..bedb4c6 100644 --- a/database/database_test.go +++ b/database/database_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "log" "os" "reflect" @@ -22,7 +21,7 @@ import ( ) func TestMain(m *testing.M) { - testDir, err := ioutil.TempDir("", "portbase-database-testing-") + testDir, err := os.MkdirTemp("", "portbase-database-testing-") if err != nil { panic(err) } diff --git a/database/doc.go b/database/doc.go index 3c8ff8b..1e1e6a5 100644 --- a/database/doc.go +++ b/database/doc.go @@ -1,63 +1,62 @@ /* Package database provides a universal interface for interacting with the database. -A Lazy Database +# A Lazy Database The database system can handle Go structs as well as serialized data by the dsd package. While data is in transit within the system, it does not know which form it currently has. Only when it reaches its destination, it must ensure that it is either of a certain type or dump it. -Record Interface +# Record Interface The database system uses the Record interface to transparently handle all types of structs that get saved in the database. Structs include the Base struct to fulfill most parts of the Record interface. Boilerplate Code: - type Example struct { - record.Base - sync.Mutex + type Example struct { + record.Base + sync.Mutex - Name string - Score int - } + Name string + Score int + } - var ( - db = database.NewInterface(nil) - ) + var ( + db = database.NewInterface(nil) + ) - // GetExample gets an Example from the database. - func GetExample(key string) (*Example, error) { - r, err := db.Get(key) - if err != nil { - return nil, err - } + // GetExample gets an Example from the database. + func GetExample(key string) (*Example, error) { + r, err := db.Get(key) + if err != nil { + return nil, err + } - // unwrap - if r.IsWrapped() { - // only allocate a new struct, if we need it - new := &Example{} - err = record.Unwrap(r, new) - if err != nil { - return nil, err - } - return new, nil - } + // unwrap + if r.IsWrapped() { + // only allocate a new struct, if we need it + new := &Example{} + err = record.Unwrap(r, new) + if err != nil { + return nil, err + } + return new, nil + } - // or adjust type - new, ok := r.(*Example) - if !ok { - return nil, fmt.Errorf("record not of type *Example, but %T", r) - } - return new, nil - } + // or adjust type + new, ok := r.(*Example) + if !ok { + return nil, fmt.Errorf("record not of type *Example, but %T", r) + } + return new, nil + } - func (e *Example) Save() error { - return db.Put(e) - } - - func (e *Example) SaveAs(key string) error { - e.SetKey(key) - return db.PutNew(e) - } + func (e *Example) Save() error { + return db.Put(e) + } + func (e *Example) SaveAs(key string) error { + e.SetKey(key) + return db.PutNew(e) + } */ package database diff --git a/database/query/parser.go b/database/query/parser.go index 988ea59..b6abd39 100644 --- a/database/query/parser.go +++ b/database/query/parser.go @@ -14,6 +14,7 @@ type snippet struct { } // ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces. +// //nolint:gocognit func ParseQuery(query string) (*Query, error) { snippets, err := extractSnippets(query) diff --git a/database/registry.go b/database/registry.go index 7c68b59..2694c14 100644 --- a/database/registry.go +++ b/database/registry.go @@ -4,7 +4,6 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" "os" "path" "regexp" @@ -115,7 +114,7 @@ func loadRegistry() error { // read file filePath := path.Join(rootStructure.Path, registryFileName) - data, err := ioutil.ReadFile(filePath) + data, err := os.ReadFile(filePath) if err != nil { if os.IsNotExist(err) { return nil @@ -150,7 +149,7 @@ func saveRegistry(lock bool) error { // write file // TODO: write atomically (best effort) filePath := path.Join(rootStructure.Path, registryFileName) - return ioutil.WriteFile(filePath, data, 0o0600) + return os.WriteFile(filePath, data, 0o0600) } func registryWriter() { diff --git a/database/storage/badger/badger_test.go b/database/storage/badger/badger_test.go index 9cff49f..40bc8f0 100644 --- a/database/storage/badger/badger_test.go +++ b/database/storage/badger/badger_test.go @@ -2,7 +2,6 @@ package badger import ( "context" - "io/ioutil" "os" "reflect" "sync" @@ -41,7 +40,7 @@ type TestRecord struct { //nolint:maligned func TestBadger(t *testing.T) { t.Parallel() - testDir, err := ioutil.TempDir("", "testing-") + testDir, err := os.MkdirTemp("", "testing-") if err != nil { t.Fatal(err) } diff --git a/database/storage/bbolt/bbolt_test.go b/database/storage/bbolt/bbolt_test.go index 5e9b302..9fdfca8 100644 --- a/database/storage/bbolt/bbolt_test.go +++ b/database/storage/bbolt/bbolt_test.go @@ -2,7 +2,6 @@ package bbolt import ( "context" - "io/ioutil" "os" "reflect" "sync" @@ -43,7 +42,7 @@ type TestRecord struct { //nolint:maligned func TestBBolt(t *testing.T) { t.Parallel() - testDir, err := ioutil.TempDir("", "testing-") + testDir, err := os.MkdirTemp("", "testing-") if err != nil { t.Fatal(err) } diff --git a/database/storage/fstree/fstree.go b/database/storage/fstree/fstree.go index cf6c56c..dfc9e17 100644 --- a/database/storage/fstree/fstree.go +++ b/database/storage/fstree/fstree.go @@ -8,7 +8,6 @@ import ( "context" "errors" "fmt" - "io/ioutil" "os" "path/filepath" "runtime" @@ -88,7 +87,7 @@ func (fst *FSTree) Get(key string) (record.Record, error) { return nil, err } - data, err := ioutil.ReadFile(dstPath) + data, err := os.ReadFile(dstPath) if err != nil { if os.IsNotExist(err) { return nil, storage.ErrNotFound @@ -210,7 +209,7 @@ func (fst *FSTree) queryExecutor(walkRoot string, queryIter *iterator.Iterator, } // read file - data, err := ioutil.ReadFile(path) + data, err := os.ReadFile(path) if err != nil { if os.IsNotExist(err) { return nil diff --git a/formats/dsd/http.go b/formats/dsd/http.go index 8653785..7b2e5c5 100644 --- a/formats/dsd/http.go +++ b/formats/dsd/http.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "mime" "net/http" ) @@ -33,7 +32,7 @@ func LoadFromHTTPResponse(resp *http.Response, t interface{}) (format uint8, err func loadFromHTTP(body io.Reader, mimeType string, t interface{}) (format uint8, err error) { // Read full body. - data, err := ioutil.ReadAll(body) + data, err := io.ReadAll(body) if err != nil { return 0, fmt.Errorf("dsd: failed to read http body: %w", err) } @@ -90,7 +89,7 @@ func DumpToHTTPRequest(r *http.Request, t interface{}, format uint8) error { // Set body. r.Header.Set("Content-Type", mimeType) - r.Body = ioutil.NopCloser(bytes.NewReader(data)) + r.Body = io.NopCloser(bytes.NewReader(data)) return nil } diff --git a/go.mod b/go.mod index c649047..32f11c3 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/dgraph-io/badger v1.6.2 github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/fxamacker/cbor/v2 v2.4.0 - github.com/gofrs/uuid v4.2.0+incompatible + github.com/gofrs/uuid v4.3.0+incompatible github.com/golang/glog v1.0.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/mux v1.8.0 @@ -21,6 +21,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 github.com/hashicorp/go-version v1.6.0 github.com/mitchellh/copystructure v1.2.0 + github.com/safing/jess v0.3.0 // indirect github.com/seehuhn/fortuna v1.0.1 github.com/shirou/gopsutil v3.21.11+incompatible github.com/stretchr/testify v1.8.0 @@ -34,7 +35,7 @@ require ( go.etcd.io/bbolt v1.3.6 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220209214540-3681064d5158 + golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/protobuf v1.27.1 // indirect ) diff --git a/go.sum b/go.sum index 18da244..a22019d 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,16 @@ +github.com/AlecAivazis/survey/v2 v2.3.6/go.mod h1:4AuI9b7RjAR+G7v9+C4YSlX/YL3K3cWNXgWXOhllqvI= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/VictoriaMetrics/metrics v1.22.2 h1:A6LsNidYwkAHetxsvNFaUWjtzu5ltdgNEoS6i7Bn+6I= github.com/VictoriaMetrics/metrics v1.22.2/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= +github.com/aead/ecdh v0.2.0 h1:pYop54xVaq/CEREFEcukHRZfTdjiWvYIsZDXXrBapQQ= +github.com/aead/ecdh v0.2.0/go.mod h1:a9HHtXuSo8J1Js1MwLQx2mBhkXMT6YwUmVVEY4tTB8U= github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6 h1:5L8Mj9Co9sJVgW3TpYk2gxGJnDjsYuboNTcRmbtGKGs= github.com/aead/serpent v0.0.0-20160714141033-fba169763ea6/go.mod h1:3HgLJ9d18kXMLQlJvIY3+FszZYMxCz8WfE2MQ7hDY0w= +github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= @@ -20,6 +25,10 @@ github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= +github.com/danieljoos/wincred v1.1.2/go.mod h1:GijpziifJoIBfYh+S7BbkdUTU4LfM+QnGqR5Vl2tAx0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -37,8 +46,12 @@ github.com/fxamacker/cbor/v2 v2.4.0 h1:ri0ArlOR+5XunOP8CRUowT0pSJOwhW098ZCUyskZD github.com/fxamacker/cbor/v2 v2.4.0/go.mod h1:TA1xS00nchWmaBnEIxPSE5oHLuJBAVvqrtAnWBwBCVo= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.2.0+incompatible h1:yyYWMnhkhrKwwr8gAOcOCYxOOscHgDS9yZgBrnJfGa0= github.com/gofrs/uuid v4.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gofrs/uuid v4.3.0+incompatible h1:CaSVZxm5B+7o45rtab4jC2G37WGYX1zQfuU2i6DSvnc= +github.com/gofrs/uuid v4.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= @@ -60,19 +73,30 @@ github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mr-tron/base58 v1.2.0 h1:T/HDJBh4ZCPbU39/+c3rRvE0uKBQlU27+QI8LJ4t64o= +github.com/mr-tron/base58 v1.2.0/go.mod h1:BinMc/sQntlIE1frQmRFPUoPA1Zkr8VRgBdjWI2mNwc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -80,6 +104,12 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/safing/jess v0.3.0 h1:NxerZE5Vrludn00gyR4VeZaNjbDYq/qBzmcV3SLfjd4= +github.com/safing/jess v0.3.0/go.mod h1:JbYsPk5iJZx0OXDZeMcjS9qEdkGVUg+DCA8Fw2LdN9s= +github.com/safing/portbase v0.15.2/go.mod h1:5bHi99fz7Hh/wOsZUOI631WF9ePSHk57c4fdlOMS91Y= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/seehuhn/fortuna v1.0.1 h1:lu9+CHsmR0bZnx5Ay646XvCSRJ8PJTi5UYJwDBX68H0= github.com/seehuhn/fortuna v1.0.1/go.mod h1:LX8ubejCnUoT/hX+1aKUtbKls2H6DRkqzkc7TdR3iis= github.com/seehuhn/sha256d v1.0.0 h1:TXTsAuEWr02QjRm153Fnvvb6fXXDo7Bmy1FizxarGYw= @@ -91,14 +121,18 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/cobra v1.5.0/go.mod h1:dWXEIy2H428czQCjInthrTRUg7yKbok+2Qi/yBIJoUM= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= @@ -132,29 +166,47 @@ github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcY github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zalando/go-keyring v0.2.1/go.mod h1:g63M2PPn0w5vjmEbwAX3ib5I+41zdm4esSETOn9Y6Dw= go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be h1:fmw3UbQh+nxngCAHrDCCztao/kbYFnWjoqop8dHx05A= +golang.org/x/crypto v0.0.0-20220926161630-eccd6366d1be/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd h1:O7DYs+zxREGLKzKoMQrtrEacpb0ZVXA5rIwylE2Xchk= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210816074244-15123e1e1f71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158 h1:rm+CHSpPEEW2IsXUib1ThaHIjuBVZjxNgSKmBLFfD4c= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI= +golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20220919170432-7a66f970e087/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -168,6 +220,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/metrics/api.go b/metrics/api.go index 974140e..859e1bb 100644 --- a/metrics/api.go +++ b/metrics/api.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "io/ioutil" "net/http" "time" @@ -111,7 +110,7 @@ func writeMetricsTo(ctx context.Context, url string) error { } // Get and return error. - body, _ := ioutil.ReadAll(resp.Body) + body, _ := io.ReadAll(resp.Body) return fmt.Errorf( "got %s while writing metrics to %s: %s", resp.Status, diff --git a/modules/mgmt.go b/modules/mgmt.go index adb460d..7bacddb 100644 --- a/modules/mgmt.go +++ b/modules/mgmt.go @@ -55,14 +55,13 @@ func (m *Module) EnabledAsDependency() bool { // // Example: // -// EnableModuleManagement(func(m *modules.Module) { -// // some module has changed ... -// // do what ever you like -// -// // Run the built-in module management -// modules.ManageModules() -// }) +// EnableModuleManagement(func(m *modules.Module) { +// // some module has changed ... +// // do what ever you like // +// // Run the built-in module management +// modules.ManageModules() +// }) func EnableModuleManagement(changeNotifyFn func(*Module)) bool { if moduleMgmtEnabled.SetToIf(false, true) { modulesChangeNotifyFn = changeNotifyFn diff --git a/modules/subsystems/registry.go b/modules/subsystems/registry.go index 2914243..50b472f 100644 --- a/modules/subsystems/registry.go +++ b/modules/subsystems/registry.go @@ -125,11 +125,11 @@ func (mng *Manager) Get(keyOrPrefix string) ([]record.Record, error) { // you. Pass a nil option to force enable. // // TODO(ppacher): IMHO the subsystem package is not responsible of registering -// the "toggle option". This would also remove runtime -// dependency to the config package. Users should either pass -// the BoolOptionFunc and the expertise/release level directly -// or just pass the configuration key so those information can -// be looked up by the registry. +// the "toggle option". This would also remove runtime +// dependency to the config package. Users should either pass +// the BoolOptionFunc and the expertise/release level directly +// or just pass the configuration key so those information can +// be looked up by the registry. func (mng *Manager) Register(id, name, description string, module *modules.Module, configKeySpace string, option *config.Option) error { mng.l.Lock() defer mng.l.Unlock() diff --git a/modules/subsystems/subsystems_test.go b/modules/subsystems/subsystems_test.go index ab632e5..e6ba837 100644 --- a/modules/subsystems/subsystems_test.go +++ b/modules/subsystems/subsystems_test.go @@ -1,7 +1,6 @@ package subsystems import ( - "io/ioutil" "os" "testing" "time" @@ -14,7 +13,7 @@ import ( func TestSubsystems(t *testing.T) { //nolint:paralleltest // Too much interference expected. // tmp dir for data root (db & config) - tmpDir, err := ioutil.TempDir("", "portbase-testing-") + tmpDir, err := os.MkdirTemp("", "portbase-testing-") // initialize data dir if err == nil { err = dataroot.Initialize(tmpDir, 0o0755) diff --git a/notifications/doc.go b/notifications/doc.go index 29c3b53..be18386 100644 --- a/notifications/doc.go +++ b/notifications/doc.go @@ -1,7 +1,7 @@ /* Package notifications provides a notification system. -Notification Lifecycle +# Notification Lifecycle 1. Create Notification with an ID and Message. 2. Set possible actions and save it. @@ -9,19 +9,18 @@ Notification Lifecycle Example - // create notification - n := notifications.New("update-available", "A new update is available. Restart to upgrade.") - // set actions and save - n.AddAction("later", "Later").AddAction("restart", "Restart now!").Save() - - // wait for user action - selectedAction := <-n.Response() - switch selectedAction { - case "later": - log.Infof("user wants to upgrade later.") - case "restart": - log.Infof("user wants to restart now.") - } + // create notification + n := notifications.New("update-available", "A new update is available. Restart to upgrade.") + // set actions and save + n.AddAction("later", "Later").AddAction("restart", "Restart now!").Save() + // wait for user action + selectedAction := <-n.Response() + switch selectedAction { + case "later": + log.Infof("user wants to upgrade later.") + case "restart": + log.Infof("user wants to restart now.") + } */ package notifications diff --git a/runtime/singe_record_provider.go b/runtime/singe_record_provider.go index 6f232b5..c941f58 100644 --- a/runtime/singe_record_provider.go +++ b/runtime/singe_record_provider.go @@ -15,17 +15,16 @@ type singleRecordReader struct { // // Example: // -// type MyValue struct { -// record.Base -// Value string -// } -// r := new(MyValue) -// pushUpdate, _ := runtime.Register("my/key", ProvideRecord(r)) -// r.Lock() -// r.Value = "foobar" -// pushUpdate(r) -// r.Unlock() -// +// type MyValue struct { +// record.Base +// Value string +// } +// r := new(MyValue) +// pushUpdate, _ := runtime.Register("my/key", ProvideRecord(r)) +// r.Lock() +// r.Value = "foobar" +// pushUpdate(r) +// r.Unlock() func ProvideRecord(r record.Record) ValueProvider { return &singleRecordReader{r} } diff --git a/template/module_test.go b/template/module_test.go index 30c9bb8..8ab51f0 100644 --- a/template/module_test.go +++ b/template/module_test.go @@ -2,7 +2,6 @@ package template import ( "fmt" - "io/ioutil" "os" "testing" @@ -19,7 +18,7 @@ func TestMain(m *testing.M) { module.Enable() // tmp dir for data root (db & config) - tmpDir, err := ioutil.TempDir("", "portbase-testing-") + tmpDir, err := os.MkdirTemp("", "portbase-testing-") if err != nil { fmt.Fprintf(os.Stderr, "failed to create tmp dir: %s\n", err) os.Exit(1) diff --git a/updater/fetch.go b/updater/fetch.go index adad517..753d909 100644 --- a/updater/fetch.go +++ b/updater/fetch.go @@ -3,7 +3,9 @@ package updater import ( "bytes" "context" + "errors" "fmt" + "hash" "io" "net/http" "net/url" @@ -12,6 +14,8 @@ import ( "path/filepath" "time" + "github.com/safing/jess/filesig" + "github.com/safing/jess/lhash" "github.com/safing/portbase/log" "github.com/safing/portbase/utils/renameio" ) @@ -33,6 +37,31 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client, return fmt.Errorf("could not create updates folder: %s", dirPath) } + // If verification is enabled, download signature first. + var ( + verifiedHash *lhash.LabeledHash + sigFileData []byte + ) + if rv.resource.VerificationOptions != nil { + verifiedHash, sigFileData, err = reg.fetchAndVerifySigFile( + ctx, client, + rv.resource.VerificationOptions, + rv.versionedSigPath(), rv.SigningMetadata(), + tries, + ) + + if err != nil { + switch rv.resource.VerificationOptions.DownloadPolicy { + case SignaturePolicyRequire: + return fmt.Errorf("signature verification failed: %w", err) + case SignaturePolicyWarn: + log.Warningf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err) + case SignaturePolicyDisable: + log.Debugf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err) + } + } + } + // open file for writing atomicFile, err := renameio.TempFile(reg.tmpDir.Path, rv.storagePath()) if err != nil { @@ -49,8 +78,16 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client, _ = resp.Body.Close() }() - // download and write file - n, err := io.Copy(atomicFile, resp.Body) + // Write to the hasher at the same time, if needed. + var hasher hash.Hash + var writeDst io.Writer = atomicFile + if verifiedHash != nil { + hasher = verifiedHash.Algorithm().RawHasher() + writeDst = io.MultiWriter(hasher, atomicFile) + } + + // Download and write file. + n, err := io.Copy(writeDst, resp.Body) if err != nil { return fmt.Errorf("failed to download %q: %w", downloadURL, err) } @@ -58,6 +95,42 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client, return fmt.Errorf("failed to finish download of %q: written %d out of %d bytes", downloadURL, n, resp.ContentLength) } + // Before file is finalized, check if hash, if available. + if hasher != nil { + downloadDigest := hasher.Sum(nil) + if verifiedHash.EqualRaw(downloadDigest) { + log.Infof("%s: verified signature of %s", reg.Name, downloadURL) + } else { + switch rv.resource.VerificationOptions.DownloadPolicy { + case SignaturePolicyRequire: + return errors.New("file does not match signed checksum") + case SignaturePolicyWarn: + log.Warningf("%s: checksum does not match file from %s", reg.Name, downloadURL) + case SignaturePolicyDisable: + log.Debugf("%s: checksum does not match file from %s", reg.Name, downloadURL) + } + + // Reset hasher to signal that the sig should not be written. + hasher = nil + } + } + + // Write signature file, if we have one and if verification succeeded. + if len(sigFileData) > 0 && hasher != nil { + sigFilePath := rv.storagePath() + filesig.Extension + err := os.WriteFile(sigFilePath, sigFileData, 0o0644) //nolint:gosec + if err != nil { + switch rv.resource.VerificationOptions.DownloadPolicy { + case SignaturePolicyRequire: + return fmt.Errorf("failed to write signature file %s: %w", sigFilePath, err) + case SignaturePolicyWarn: + log.Warningf("%s: failed to write signature file %s: %s", reg.Name, sigFilePath, err) + case SignaturePolicyDisable: + log.Debugf("%s: failed to write signature file %s: %s", reg.Name, sigFilePath, err) + } + } + } + // finalize file err = atomicFile.CloseAtomicallyReplace() if err != nil { @@ -76,12 +149,140 @@ func (reg *ResourceRegistry) fetchFile(ctx context.Context, client *http.Client, return nil } -func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client, downloadPath string, tries int) ([]byte, error) { +func (reg *ResourceRegistry) fetchMissingSig(ctx context.Context, client *http.Client, rv *ResourceVersion, tries int) error { // backoff when retrying if tries > 0 { select { case <-ctx.Done(): - return nil, nil // module is shutting down + return nil // module is shutting down + case <-time.After(time.Duration(tries*tries) * time.Second): + } + } + + // Check destination dir. + dirPath := filepath.Dir(rv.storagePath()) + err := reg.storageDir.EnsureAbsPath(dirPath) + if err != nil { + return fmt.Errorf("could not create updates folder: %s", dirPath) + } + + // Download and verify the missing signature. + verifiedHash, sigFileData, err := reg.fetchAndVerifySigFile( + ctx, client, + rv.resource.VerificationOptions, + rv.versionedSigPath(), rv.SigningMetadata(), + tries, + ) + if err != nil { + switch rv.resource.VerificationOptions.DownloadPolicy { + case SignaturePolicyRequire: + return fmt.Errorf("signature verification failed: %w", err) + case SignaturePolicyWarn: + log.Warningf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err) + case SignaturePolicyDisable: + log.Debugf("%s: failed to verify downloaded signature of %s: %s", reg.Name, rv.versionedPath(), err) + } + return nil + } + + // Check if the signature matches the resource file. + ok, err := verifiedHash.MatchesFile(rv.storagePath()) + if err != nil { + switch rv.resource.VerificationOptions.DownloadPolicy { + case SignaturePolicyRequire: + return fmt.Errorf("error while verifying resource file: %w", err) + case SignaturePolicyWarn: + log.Warningf("%s: error while verifying resource file %s", reg.Name, rv.storagePath()) + case SignaturePolicyDisable: + log.Debugf("%s: error while verifying resource file %s", reg.Name, rv.storagePath()) + } + return nil + } + if !ok { + switch rv.resource.VerificationOptions.DownloadPolicy { + case SignaturePolicyRequire: + return errors.New("resource file does not match signed checksum") + case SignaturePolicyWarn: + log.Warningf("%s: checksum does not match resource file from %s", reg.Name, rv.storagePath()) + case SignaturePolicyDisable: + log.Debugf("%s: checksum does not match resource file from %s", reg.Name, rv.storagePath()) + } + return nil + } + + // Write signature file. + err = os.WriteFile(rv.storageSigPath(), sigFileData, 0o0644) //nolint:gosec + if err != nil { + switch rv.resource.VerificationOptions.DownloadPolicy { + case SignaturePolicyRequire: + return fmt.Errorf("failed to write signature file %s: %w", rv.storageSigPath(), err) + case SignaturePolicyWarn: + log.Warningf("%s: failed to write signature file %s: %s", reg.Name, rv.storageSigPath(), err) + case SignaturePolicyDisable: + log.Debugf("%s: failed to write signature file %s: %s", reg.Name, rv.storageSigPath(), err) + } + } + + log.Infof("%s: fetched %s (stored to %s)", reg.Name, rv.versionedSigPath(), rv.storageSigPath()) + return nil +} + +func (reg *ResourceRegistry) fetchAndVerifySigFile(ctx context.Context, client *http.Client, verifOpts *VerificationOptions, sigFilePath string, requiredMetadata map[string]string, tries int) (*lhash.LabeledHash, []byte, error) { + // Download signature file. + resp, _, err := reg.makeRequest(ctx, client, sigFilePath, tries) + if err != nil { + return nil, nil, err + } + defer func() { + _ = resp.Body.Close() + }() + sigFileData, err := io.ReadAll(resp.Body) + if err != nil { + return nil, nil, err + } + + // Extract all signatures. + sigs, err := filesig.ParseSigFile(sigFileData) + switch { + case len(sigs) == 0 && err != nil: + return nil, nil, fmt.Errorf("failed to parse signature file: %w", err) + case len(sigs) == 0: + return nil, nil, errors.New("no signatures found in signature file") + case err != nil: + return nil, nil, fmt.Errorf("failed to parse signature file: %w", err) + } + + // Verify all signatures. + var verifiedHash *lhash.LabeledHash + for _, sig := range sigs { + fd, err := filesig.VerifyFileData( + sig, + requiredMetadata, + verifOpts.TrustStore, + ) + if err != nil { + return nil, sigFileData, err + } + + // Save or check verified hash. + if verifiedHash == nil { + verifiedHash = fd.FileHash() + } else if !fd.FileHash().Equal(verifiedHash) { + // Return an error if two valid hashes mismatch. + // For simplicity, all hash algorithms must be the same for now. + return nil, sigFileData, errors.New("file hashes from different signatures do not match") + } + } + + return verifiedHash, sigFileData, nil +} + +func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client, downloadPath string, tries int) (fileData []byte, downloadedFrom string, err error) { + // backoff when retrying + if tries > 0 { + select { + case <-ctx.Done(): + return nil, "", nil // module is shutting down case <-time.After(time.Duration(tries*tries) * time.Second): } } @@ -89,7 +290,7 @@ func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client, // start file download resp, downloadURL, err := reg.makeRequest(ctx, client, downloadPath, tries) if err != nil { - return nil, err + return nil, downloadURL, err } defer func() { _ = resp.Body.Close() @@ -99,13 +300,13 @@ func (reg *ResourceRegistry) fetchData(ctx context.Context, client *http.Client, buf := bytes.NewBuffer(make([]byte, 0, resp.ContentLength)) n, err := io.Copy(buf, resp.Body) if err != nil { - return nil, fmt.Errorf("failed to download %q: %w", downloadURL, err) + return nil, downloadURL, fmt.Errorf("failed to download %q: %w", downloadURL, err) } if resp.ContentLength != n { - return nil, fmt.Errorf("failed to finish download of %q: written %d out of %d bytes", downloadURL, n, resp.ContentLength) + return nil, downloadURL, fmt.Errorf("failed to finish download of %q: written %d out of %d bytes", downloadURL, n, resp.ContentLength) } - return buf.Bytes(), nil + return buf.Bytes(), downloadURL, nil } func (reg *ResourceRegistry) makeRequest(ctx context.Context, client *http.Client, downloadPath string, tries int) (resp *http.Response, downloadURL string, err error) { @@ -121,7 +322,7 @@ func (reg *ResourceRegistry) makeRequest(ctx context.Context, client *http.Clien downloadURL = u.String() // create request - req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, http.NoBody) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, http.NoBody) if err != nil { return nil, "", fmt.Errorf("failed to create request for %q: %w", downloadURL, err) } diff --git a/updater/file.go b/updater/file.go index 8c78978..2eb1957 100644 --- a/updater/file.go +++ b/updater/file.go @@ -7,6 +7,7 @@ import ( semver "github.com/hashicorp/go-version" + "github.com/safing/jess/filesig" "github.com/safing/portbase/log" "github.com/safing/portbase/utils" ) @@ -45,6 +46,42 @@ func (file *File) Path() string { return file.storagePath } +// SigningMetadata returns the metadata to be included in signatures. +func (file *File) SigningMetadata() map[string]string { + return map[string]string{ + "id": file.Identifier(), + "version": file.Version(), + } +} + +// Verify verifies the given file. +func (file *File) Verify() ([]*filesig.FileData, error) { + // Check if verification is configured. + if file.resource.VerificationOptions == nil { + return nil, ErrVerificationNotConfigured + } + + // Verify file. + fileData, err := filesig.VerifyFile( + file.storagePath, + file.storagePath+filesig.Extension, + file.SigningMetadata(), + file.resource.VerificationOptions.TrustStore, + ) + if err != nil { + switch file.resource.VerificationOptions.DiskLoadPolicy { + case SignaturePolicyRequire: + return nil, err + case SignaturePolicyWarn: + log.Warningf("%s: failed to verify %s: %s", file.resource.registry.Name, file.storagePath, err) + case SignaturePolicyDisable: + log.Debugf("%s: failed to verify %s: %s", file.resource.registry.Name, file.storagePath, err) + } + } + + return fileData, nil +} + // Blacklist notifies the update system that this file is somehow broken, and should be ignored from now on, until restarted. func (file *File) Blacklist() error { return file.resource.Blacklist(file.version.VersionNumber) diff --git a/updater/get.go b/updater/get.go index b04dbbc..e235e00 100644 --- a/updater/get.go +++ b/updater/get.go @@ -11,8 +11,9 @@ import ( // Errors returned by the updater package. var ( - ErrNotFound = errors.New("the requested file could not be found") - ErrNotAvailableLocally = errors.New("the requested file is not available locally") + ErrNotFound = errors.New("the requested file could not be found") + ErrNotAvailableLocally = errors.New("the requested file is not available locally") + ErrVerificationNotConfigured = errors.New("verification not configured for this resource") ) // GetFile returns the selected (mostly newest) file with the given @@ -29,6 +30,14 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) { // check if file is available locally if file.version.Available { file.markActiveWithLocking() + + // Verify file, if configured. + _, err := file.Verify() + if err != nil && !errors.Is(err, ErrVerificationNotConfigured) { + // TODO: If verification is required, try deleting the resource and downloading it again. + return nil, fmt.Errorf("failed to verify file: %w", err) + } + return file, nil } @@ -52,6 +61,8 @@ func (reg *ResourceRegistry) GetFile(identifier string) (*File, error) { log.Tracef("%s: failed to download %s: %s, retrying (%d)", reg.Name, file.versionedPath, err, tries+1) } else { file.markActiveWithLocking() + + // TODO: We just download the file - should we verify it again? return file, nil } } diff --git a/updater/indexes.go b/updater/indexes.go index 685a849..35491b2 100644 --- a/updater/indexes.go +++ b/updater/indexes.go @@ -1,13 +1,106 @@ package updater +import ( + "encoding/json" + "errors" + "fmt" + "time" +) + +const ( + baseIndexExtension = ".json" + v2IndexExtension = ".v2.json" +) + // Index describes an index file pulled by the updater. type Index struct { // Path is the path to the index file // on the update server. Path string + // Channel holds the release channel name of the index. + // It must match the filename without extension. + Channel string + // PreRelease signifies that all versions of this index should be marked as // pre-releases, no matter if the versions actually have a pre-release tag or // not. PreRelease bool + + // LastRelease holds the time of the last seen release of this index. + LastRelease time.Time +} + +// IndexFile represents an index file. +type IndexFile struct { + Channel string + Published time.Time + + Releases map[string]string +} + +var ( + // ErrIndexChecksumMismatch is returned when an index does not match its + // signed checksum. + ErrIndexChecksumMismatch = errors.New("index checksum does mot match signature") + + // ErrIndexFromFuture is returned when an index is parsed with a + // Published timestamp that lies in the future. + ErrIndexFromFuture = errors.New("index is from the future") + + // ErrIndexIsOlder is returned when an index is parsed with an older + // Published timestamp than the current Published timestamp. + ErrIndexIsOlder = errors.New("index is older than the current one") + + // ErrIndexChannelMismatch is returned when an index is parsed with a + // different channel that the expected one. + ErrIndexChannelMismatch = errors.New("index does not match the expected channel") +) + +// ParseIndexFile parses an index file and checks if it is valid. +func ParseIndexFile(indexData []byte, channel string, lastIndexRelease time.Time) (*IndexFile, error) { + // Load into struct. + indexFile := &IndexFile{} + err := json.Unmarshal(indexData, indexFile) + if err != nil { + return nil, fmt.Errorf("failed to parse signed index data: %w", err) + } + + // Fallback to old format if there are no releases and no channel is defined. + // TODO: Remove in v1 + if len(indexFile.Releases) == 0 && indexFile.Channel == "" { + return loadOldIndexFormat(indexData, channel) + } + + // Check the index metadata. + switch { + case !indexFile.Published.IsZero() && time.Now().Before(indexFile.Published): + return indexFile, ErrIndexFromFuture + + case !indexFile.Published.IsZero() && + !lastIndexRelease.IsZero() && + lastIndexRelease.After(indexFile.Published): + return indexFile, ErrIndexIsOlder + + case channel != "" && + indexFile.Channel != "" && + channel != indexFile.Channel: + return indexFile, ErrIndexChannelMismatch + } + + return indexFile, nil +} + +func loadOldIndexFormat(indexData []byte, channel string) (*IndexFile, error) { + releases := make(map[string]string) + err := json.Unmarshal(indexData, &releases) + if err != nil { + return nil, err + } + + return &IndexFile{ + Channel: channel, + // Do NOT define `Published`, as this would break the "is newer" check. + Releases: releases, + }, nil } diff --git a/updater/indexes_test.go b/updater/indexes_test.go new file mode 100644 index 0000000..a85046c --- /dev/null +++ b/updater/indexes_test.go @@ -0,0 +1,57 @@ +package updater + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +var ( + oldFormat = `{ + "all/ui/modules/assets.zip": "0.3.0", + "all/ui/modules/portmaster.zip": "0.2.4", + "linux_amd64/core/portmaster-core": "0.8.13" +}` + + newFormat = `{ + "Channel": "stable", + "Published": "2022-01-02T00:00:00Z", + "Releases": { + "all/ui/modules/assets.zip": "0.3.0", + "all/ui/modules/portmaster.zip": "0.2.4", + "linux_amd64/core/portmaster-core": "0.8.13" + } +}` + + formatTestChannel = "stable" + formatTestReleases = map[string]string{ + "all/ui/modules/assets.zip": "0.3.0", + "all/ui/modules/portmaster.zip": "0.2.4", + "linux_amd64/core/portmaster-core": "0.8.13", + } +) + +func TestIndexParsing(t *testing.T) { + t.Parallel() + + lastRelease, err := time.Parse(time.RFC3339, "2022-01-01T00:00:00Z") + if err != nil { + t.Fatal(err) + } + + oldIndexFile, err := ParseIndexFile([]byte(oldFormat), formatTestChannel, lastRelease) + if err != nil { + t.Fatal(err) + } + + newIndexFile, err := ParseIndexFile([]byte(newFormat), formatTestChannel, lastRelease) + if err != nil { + t.Fatal(err) + } + + assert.Equal(t, formatTestChannel, oldIndexFile.Channel, "channel should be the same") + assert.Equal(t, formatTestChannel, newIndexFile.Channel, "channel should be the same") + assert.Equal(t, formatTestReleases, oldIndexFile.Releases, "releases should be the same") + assert.Equal(t, formatTestReleases, newIndexFile.Releases, "releases should be the same") +} diff --git a/updater/registry.go b/updater/registry.go index 6267a33..6c0fbd0 100644 --- a/updater/registry.go +++ b/updater/registry.go @@ -1,8 +1,12 @@ package updater import ( + "errors" + "fmt" "os" + "path/filepath" "runtime" + "strings" "sync" "github.com/safing/portbase/log" @@ -20,7 +24,7 @@ type ResourceRegistry struct { Name string storageDir *utils.DirStructure tmpDir *utils.DirStructure - indexes []Index + indexes []*Index resources map[string]*Resource UpdateURLs []string @@ -28,6 +32,12 @@ type ResourceRegistry struct { MandatoryUpdates []string AutoUnpack []string + // Verification holds a map of VerificationOptions assigned to their + // applicable identifier path prefix. + // Use an empty string to denote the default. + // Use empty options to disable verification for a path prefix. + Verification map[string]*VerificationOptions + // UsePreReleases signifies that pre-releases should be used when selecting a // version. Even if false, a pre-release version will still be used if it is // defined as the current version by an index. @@ -43,7 +53,12 @@ func (reg *ResourceRegistry) AddIndex(idx Index) { reg.Lock() defer reg.Unlock() - reg.indexes = append(reg.indexes, idx) + // Get channel name from path. + idx.Channel = strings.TrimSuffix( + filepath.Base(idx.Path), filepath.Ext(idx.Path), + ) + + reg.indexes = append(reg.indexes, &idx) } // Initialize initializes a raw registry struct and makes it ready for usage. @@ -76,6 +91,32 @@ func (reg *ResourceRegistry) Initialize(storageDir *utils.DirStructure) error { log.Warningf("%s: failed to create tmp dir: %s", reg.Name, err) } + // Check verification options. + if reg.Verification != nil { + for prefix, opts := range reg.Verification { + // Check if verification is disable for this prefix. + if opts == nil { + continue + } + + // If enabled, a trust store is required. + if opts.TrustStore == nil { + return fmt.Errorf("verification enabled for prefix %q, but no trust store configured", prefix) + } + + // DownloadPolicy must be equal or stricter than DiskLoadPolicy. + if opts.DiskLoadPolicy < opts.DownloadPolicy { + return errors.New("verification download policy must be equal or stricter than the disk load policy") + } + + // Warn if all policies are disabled. + if opts.DownloadPolicy == SignaturePolicyDisable && + opts.DiskLoadPolicy == SignaturePolicyDisable { + log.Warningf("%s: verification enabled for prefix %q, but all policies set to disable", reg.Name, prefix) + } + } + } + return nil } @@ -190,7 +231,7 @@ func (reg *ResourceRegistry) ResetIndexes() { reg.Lock() defer reg.Unlock() - reg.indexes = make([]Index, 0, 5) + reg.indexes = make([]*Index, 0, len(reg.indexes)) } // Cleanup removes temporary files. diff --git a/updater/registry_test.go b/updater/registry_test.go index d25ef87..945d3da 100644 --- a/updater/registry_test.go +++ b/updater/registry_test.go @@ -1,7 +1,6 @@ package updater import ( - "io/ioutil" "os" "testing" @@ -12,7 +11,7 @@ var registry *ResourceRegistry func TestMain(m *testing.M) { // setup - tmpDir, err := ioutil.TempDir("", "ci-portmaster-") + tmpDir, err := os.MkdirTemp("", "ci-portmaster-") if err != nil { panic(err) } diff --git a/updater/resource.go b/updater/resource.go index eeda5db..6195b79 100644 --- a/updater/resource.go +++ b/updater/resource.go @@ -10,7 +10,9 @@ import ( semver "github.com/hashicorp/go-version" + "github.com/safing/jess/filesig" "github.com/safing/portbase/log" + "github.com/safing/portbase/utils" ) var devVersion *semver.Version @@ -49,6 +51,9 @@ type Resource struct { // to download the latest version from the updates servers // specified in the resource registry. SelectedVersion *ResourceVersion + + // VerificationOptions holds the verification options for this resource. + VerificationOptions *VerificationOptions } // ResourceVersion represents a single version of a resource. @@ -63,6 +68,9 @@ type ResourceVersion struct { // Available indicates if this version is available locally. Available bool + // SigAvailable indicates if the signature of this version is available locally. + SigAvailable bool + // CurrentRelease indicates that this is the current release that should be // selected, if possible. CurrentRelease bool @@ -132,9 +140,7 @@ func (res *Resource) Export() *Resource { SelectedVersion: res.SelectedVersion, } // Copy Versions slice. - for i := 0; i < len(res.Versions); i++ { - export.Versions[i] = res.Versions[i] - } + copy(export.Versions, res.Versions) return export } @@ -184,9 +190,10 @@ func (res *Resource) AnyVersionAvailable() bool { func (reg *ResourceRegistry) newResource(identifier string) *Resource { return &Resource{ - registry: reg, - Identifier: identifier, - Versions: make([]*ResourceVersion, 0, 1), + registry: reg, + Identifier: identifier, + Versions: make([]*ResourceVersion, 0, 1), + VerificationOptions: reg.GetVerificationOptions(identifier), } } @@ -230,6 +237,12 @@ func (res *Resource) AddVersion(version string, available, currentRelease, preRe // set flags if available { rv.Available = true + + // If available and signatures are enabled for this resource, check if the + // signature is available. + if res.VerificationOptions != nil && utils.PathExists(rv.storageSigPath()) { + rv.SigAvailable = true + } } if currentRelease { rv.CurrentRelease = true @@ -439,8 +452,13 @@ boundarySearch: // Purge everything beyond the purge boundary. for _, rv := range res.Versions[purgeBoundary:] { - storagePath := rv.storagePath() + // Only remove if resource file is actually available. + if !rv.Available { + continue + } + // Remove resource file. + storagePath := rv.storagePath() err := os.Remove(storagePath) if err != nil { log.Warningf("%s: failed to purge resource %s v%s: %s", res.registry.Name, rv.resource.Identifier, rv.VersionNumber, err) @@ -472,10 +490,52 @@ boundarySearch: res.Versions = res.Versions[purgeBoundary:] } +// SigningMetadata returns the metadata to be included in signatures. +func (rv *ResourceVersion) SigningMetadata() map[string]string { + return map[string]string{ + "id": rv.resource.Identifier, + "version": rv.VersionNumber, + } +} + +// GetFile returns the version as a *File. +// It locks the resource for doing so. +func (rv *ResourceVersion) GetFile() *File { + rv.resource.Lock() + defer rv.resource.Unlock() + + // check for notifier + if rv.resource.notifier == nil { + // create new notifier + rv.resource.notifier = newNotifier() + } + + // create file + return &File{ + resource: rv.resource, + version: rv, + notifier: rv.resource.notifier, + versionedPath: rv.versionedPath(), + storagePath: rv.storagePath(), + } +} + +// versionedPath returns the versioned identifier. func (rv *ResourceVersion) versionedPath() string { return GetVersionedPath(rv.resource.Identifier, rv.VersionNumber) } +// versionedSigPath returns the versioned identifier of the file signature. +func (rv *ResourceVersion) versionedSigPath() string { + return GetVersionedPath(rv.resource.Identifier, rv.VersionNumber) + filesig.Extension +} + +// storagePath returns the absolute storage path. func (rv *ResourceVersion) storagePath() string { return filepath.Join(rv.resource.registry.storageDir.Path, filepath.FromSlash(rv.versionedPath())) } + +// storageSigPath returns the absolute storage path of the file signature. +func (rv *ResourceVersion) storageSigPath() string { + return rv.storagePath() + filesig.Extension +} diff --git a/updater/signing.go b/updater/signing.go new file mode 100644 index 0000000..cffd5cb --- /dev/null +++ b/updater/signing.go @@ -0,0 +1,49 @@ +package updater + +import ( + "strings" + + "github.com/safing/jess" +) + +// VerificationOptions holds options for verification of files. +type VerificationOptions struct { + TrustStore jess.TrustStore + DownloadPolicy SignaturePolicy + DiskLoadPolicy SignaturePolicy +} + +// GetVerificationOptions returns the verification options for the given identifier. +func (reg *ResourceRegistry) GetVerificationOptions(identifier string) *VerificationOptions { + if reg.Verification == nil { + return nil + } + + var ( + longestPrefix = -1 + bestMatch *VerificationOptions + ) + for prefix, opts := range reg.Verification { + if len(prefix) > longestPrefix && strings.HasPrefix(identifier, prefix) { + longestPrefix = len(prefix) + bestMatch = opts + } + } + + return bestMatch +} + +// SignaturePolicy defines behavior in case of errors. +type SignaturePolicy uint8 + +// Signature Policies. +const ( + // SignaturePolicyRequire fails on any error. + SignaturePolicyRequire = iota + + // SignaturePolicyWarn only warns on errors. + SignaturePolicyWarn + + // SignaturePolicyDisable only downloads signatures, but does not verify them. + SignaturePolicyDisable +) diff --git a/updater/storage.go b/updater/storage.go index e95895f..cfae4cf 100644 --- a/updater/storage.go +++ b/updater/storage.go @@ -2,15 +2,15 @@ package updater import ( "context" - "encoding/json" "errors" "fmt" - "io/ioutil" "net/http" "os" "path/filepath" "strings" + "github.com/safing/jess/filesig" + "github.com/safing/jess/lhash" "github.com/safing/portbase/log" "github.com/safing/portbase/utils" ) @@ -51,6 +51,11 @@ func (reg *ResourceRegistry) ScanStorage(root string) error { return nil } + // Ignore file signatures. + if strings.HasSuffix(path, filesig.Extension) { + return nil + } + // get relative path to storage relativePath, err := filepath.Rel(reg.storageDir.Path, path) if err != nil { @@ -110,39 +115,118 @@ func (reg *ResourceRegistry) LoadIndexes(ctx context.Context) error { return firstErr } -func (reg *ResourceRegistry) getIndexes() []Index { +// getIndexes returns a copy of the index. +// The indexes itself are references. +func (reg *ResourceRegistry) getIndexes() []*Index { reg.RLock() defer reg.RUnlock() - indexes := make([]Index, len(reg.indexes)) + + indexes := make([]*Index, len(reg.indexes)) copy(indexes, reg.indexes) return indexes } -func (reg *ResourceRegistry) loadIndexFile(idx Index) error { - path := filepath.FromSlash(idx.Path) - data, err := ioutil.ReadFile(filepath.Join(reg.storageDir.Path, path)) +func (reg *ResourceRegistry) loadIndexFile(idx *Index) error { + indexPath := filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)) + indexData, err := os.ReadFile(indexPath) if err != nil { - return err + return fmt.Errorf("failed to read index file %s: %w", idx.Path, err) } - releases := make(map[string]string) - err = json.Unmarshal(data, &releases) - if err != nil { - return err + // Verify signature, if enabled. + if verifOpts := reg.GetVerificationOptions(idx.Path); verifOpts != nil { + // Load and check signature. + verifiedHash, _, err := reg.loadAndVerifySigFile(verifOpts, indexPath+filesig.Extension) + if err != nil { + switch verifOpts.DiskLoadPolicy { + case SignaturePolicyRequire: + return fmt.Errorf("failed to verify signature of index %s: %w", idx.Path, err) + case SignaturePolicyWarn: + log.Warningf("%s: failed to verify signature of index %s: %s", reg.Name, idx.Path, err) + case SignaturePolicyDisable: + log.Debugf("%s: failed to verify signature of index %s: %s", reg.Name, idx.Path, err) + } + } + + // Check if signature checksum matches the index data. + if err == nil && !verifiedHash.Matches(indexData) { + switch verifOpts.DiskLoadPolicy { + case SignaturePolicyRequire: + return fmt.Errorf("index file %s does not match signature", idx.Path) + case SignaturePolicyWarn: + log.Warningf("%s: index file %s does not match signature", reg.Name, idx.Path) + case SignaturePolicyDisable: + log.Debugf("%s: index file %s does not match signature", reg.Name, idx.Path) + } + } } - if len(releases) == 0 { - log.Debugf("%s: index %s is empty", reg.Name, idx.Path) + // Parse the index file. + indexFile, err := ParseIndexFile(indexData, idx.Channel, idx.LastRelease) + if err != nil { + return fmt.Errorf("failed to parse index file %s: %w", idx.Path, err) + } + + // Update last seen release. + idx.LastRelease = indexFile.Published + + // Warn if there aren't any releases in the index. + if len(indexFile.Releases) == 0 { + log.Debugf("%s: index %s has no releases", reg.Name, idx.Path) return nil } - err = reg.AddResources(releases, false, true, idx.PreRelease) + // Add index releases to available resources. + err = reg.AddResources(indexFile.Releases, false, true, idx.PreRelease) if err != nil { log.Warningf("%s: failed to add resource: %s", reg.Name, err) } return nil } +func (reg *ResourceRegistry) loadAndVerifySigFile(verifOpts *VerificationOptions, sigFilePath string) (*lhash.LabeledHash, []byte, error) { + // Load signature file. + sigFileData, err := os.ReadFile(sigFilePath) + if err != nil { + return nil, nil, fmt.Errorf("failed to read signature file: %w", err) + } + + // Extract all signatures. + sigs, err := filesig.ParseSigFile(sigFileData) + switch { + case len(sigs) == 0 && err != nil: + return nil, nil, fmt.Errorf("failed to parse signature file: %w", err) + case len(sigs) == 0: + return nil, nil, errors.New("no signatures found in signature file") + case err != nil: + return nil, nil, fmt.Errorf("failed to parse signature file: %w", err) + } + + // Verify all signatures. + var verifiedHash *lhash.LabeledHash + for _, sig := range sigs { + fd, err := filesig.VerifyFileData( + sig, + nil, + verifOpts.TrustStore, + ) + if err != nil { + return nil, sigFileData, err + } + + // Save or check verified hash. + if verifiedHash == nil { + verifiedHash = fd.FileHash() + } else if !fd.FileHash().Equal(verifiedHash) { + // Return an error if two valid hashes mismatch. + // For simplicity, all hash algorithms must be the same for now. + return nil, sigFileData, errors.New("file hashes from different signatures do not match") + } + } + + return verifiedHash, sigFileData, nil +} + // CreateSymlinks creates a directory structure with unversioned symlinks to the given updates list. func (reg *ResourceRegistry) CreateSymlinks(symlinkRoot *utils.DirStructure) error { err := os.RemoveAll(symlinkRoot.Path) diff --git a/updater/storage_test.go b/updater/storage_test.go index eafabe4..2e4122f 100644 --- a/updater/storage_test.go +++ b/updater/storage_test.go @@ -13,7 +13,7 @@ func testLoadLatestScope(t *testing.T, basePath, filePath, expectedIdentifier, e } // touch file - err = ioutil.WriteFile(fullPath, []byte{}, 0644) + err = os.WriteFile(fullPath, []byte{}, 0644) if err != nil { t.Fatalf("could not create test file: %s\n", err) return @@ -45,7 +45,7 @@ func TestLoadLatestScope(t *testing.T) { updatesLock.Lock() defer updatesLock.Unlock() - tmpDir, err := ioutil.TempDir("", "testing_") + tmpDir, err := os.MkdirTemp("", "testing_") if err != nil { t.Fatalf("could not create test dir: %s\n", err) return diff --git a/updater/updating.go b/updater/updating.go index f045186..116db17 100644 --- a/updater/updating.go +++ b/updater/updating.go @@ -2,15 +2,16 @@ package updater import ( "context" - "encoding/json" "fmt" - "io/ioutil" "net/http" + "os" "path" "path/filepath" "strings" "sync" + "github.com/safing/jess/filesig" + "github.com/safing/jess/lhash" "github.com/safing/portbase/log" "github.com/safing/portbase/utils" ) @@ -37,38 +38,87 @@ func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error { return nil } -func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Client, idx Index) error { - var err error - var data []byte +func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Client, idx *Index) error { + var ( + // Index. + indexErr error + indexData []byte + downloadURL string - // download new index + // Signature. + sigErr error + verifiedHash *lhash.LabeledHash + sigFileData []byte + verifOpts = reg.GetVerificationOptions(idx.Path) + ) + + // Upgrade to v2 index if verification is enabled. + downloadIndexPath := idx.Path + if verifOpts != nil { + downloadIndexPath = strings.TrimSuffix(downloadIndexPath, baseIndexExtension) + v2IndexExtension + } + + // Download new index and signature. for tries := 0; tries < 3; tries++ { - data, err = reg.fetchData(ctx, client, idx.Path, tries) - if err == nil { - break + // Index and signature need to be fetched together, so that they are + // fetched from the same source. One source should always have a matching + // index and signature. Backup sources may be behind a little. + // If the signature verification fails, another source should be tried. + + // Get index data. + indexData, downloadURL, indexErr = reg.fetchData(ctx, client, downloadIndexPath, tries) + if indexErr != nil { + log.Debugf("%s: failed to fetch index %s: %s", reg.Name, downloadURL, indexErr) + continue } + + // Get signature and verify it. + if verifOpts != nil { + verifiedHash, sigFileData, sigErr = reg.fetchAndVerifySigFile( + ctx, client, + verifOpts, downloadIndexPath+filesig.Extension, nil, + tries, + ) + if sigErr != nil { + log.Debugf("%s: failed to verify signature of %s: %s", reg.Name, downloadURL, sigErr) + continue + } + + // Check if the index matches the verified hash. + if verifiedHash.Matches(indexData) { + log.Infof("%s: verified signature of %s", reg.Name, downloadURL) + } else { + sigErr = ErrIndexChecksumMismatch + log.Debugf("%s: checksum does not match file from %s", reg.Name, downloadURL) + continue + } + } + + break } - if err != nil { - return fmt.Errorf("failed to download index %s: %w", idx.Path, err) + if indexErr != nil { + return fmt.Errorf("failed to fetch index %s: %w", downloadIndexPath, indexErr) + } + if sigErr != nil { + return fmt.Errorf("failed to fetch or verify index %s signature: %w", downloadIndexPath, sigErr) } - // parse - newIndexData := make(map[string]string) - err = json.Unmarshal(data, &newIndexData) + // Parse the index file. + indexFile, err := ParseIndexFile(indexData, idx.Channel, idx.LastRelease) if err != nil { return fmt.Errorf("failed to parse index %s: %w", idx.Path, err) } // Add index data to registry. - if len(newIndexData) > 0 { + if len(indexFile.Releases) > 0 { // Check if all resources are within the indexes' authority. authoritativePath := path.Dir(idx.Path) + "/" if authoritativePath == "./" { // Fix path for indexes at the storage root. authoritativePath = "" } - cleanedData := make(map[string]string, len(newIndexData)) - for key, version := range newIndexData { + cleanedData := make(map[string]string, len(indexFile.Releases)) + for key, version := range indexFile.Releases { if strings.HasPrefix(key, authoritativePath) { cleanedData[key] = version } else { @@ -85,22 +135,34 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli log.Debugf("%s: index %s is empty", reg.Name, idx.Path) } - // check if dest dir exists + // Check if dest dir exists. indexDir := filepath.FromSlash(path.Dir(idx.Path)) err = reg.storageDir.EnsureRelPath(indexDir) if err != nil { log.Warningf("%s: failed to ensure directory for updated index %s: %s", reg.Name, idx.Path, err) } - // save index - indexPath := filepath.FromSlash(idx.Path) // Index files must be readable by portmaster-staert with user permissions in order to load the index. - err = ioutil.WriteFile(filepath.Join(reg.storageDir.Path, indexPath), data, 0o0644) //nolint:gosec + err = os.WriteFile( //nolint:gosec + filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)), + indexData, 0o0644, + ) if err != nil { log.Warningf("%s: failed to save updated index %s: %s", reg.Name, idx.Path, err) } - log.Infof("%s: updated index %s with %d entries", reg.Name, idx.Path, len(newIndexData)) + // Write signature file, if we have one. + if len(sigFileData) > 0 { + err = os.WriteFile( //nolint:gosec + filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)+filesig.Extension), + sigFileData, 0o0644, + ) + if err != nil { + log.Warningf("%s: failed to save updated index signature %s: %s", reg.Name, idx.Path+filesig.Extension, err) + } + } + + log.Infof("%s: updated index %s with %d entries", reg.Name, idx.Path, len(indexFile.Releases)) return nil } @@ -108,6 +170,7 @@ func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Cli func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { // create list of downloads var toUpdate []*ResourceVersion + var missingSigs []*ResourceVersion reg.RLock() for _, res := range reg.resources { res.Lock() @@ -119,8 +182,15 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { // add all non-available and eligible versions to update queue for _, rv := range res.Versions { - if !rv.Available && rv.CurrentRelease { + switch { + case !rv.CurrentRelease: + // We are not interested in older releases. + case !rv.Available: + // File is not available. toUpdate = append(toUpdate, rv) + case !rv.SigAvailable && res.VerificationOptions != nil: + // File signature is not available and verification is enabled. + missingSigs = append(missingSigs, rv) } } } @@ -130,7 +200,7 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { reg.RUnlock() // nothing to update - if len(toUpdate) == 0 { + if len(toUpdate) == 0 && len(missingSigs) == 0 { log.Infof("%s: everything up to date", reg.Name) return nil } @@ -141,10 +211,10 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { } // download updates - log.Infof("%s: starting to download %d updates parallel", reg.Name, len(toUpdate)) + log.Infof("%s: starting to download %d updates in parallel", reg.Name, len(toUpdate)+len(missingSigs)) var wg sync.WaitGroup - wg.Add(len(toUpdate)) + wg.Add(len(toUpdate) + len(missingSigs)) client := &http.Client{} for idx := range toUpdate { @@ -171,6 +241,30 @@ func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context) error { }(toUpdate[idx]) } + for idx := range missingSigs { + go func(rv *ResourceVersion) { + var err error + + defer wg.Done() + defer func() { + if x := recover(); x != nil { + log.Errorf("%s: %s: captured panic: %s", reg.Name, rv.resource.Identifier, x) + } + }() + + for tries := 0; tries < 3; tries++ { + err = reg.fetchMissingSig(ctx, client, rv, tries) + if err == nil { + rv.SigAvailable = true + return + } + } + if err != nil { + log.Warningf("%s: failed to download missing sig of %s version %s: %s", reg.Name, rv.resource.Identifier, rv.VersionNumber, err) + } + }(missingSigs[idx]) + } + wg.Wait() log.Infof("%s: finished downloading updates", reg.Name) diff --git a/utils/fs.go b/utils/fs.go index e0873a7..87d2d59 100644 --- a/utils/fs.go +++ b/utils/fs.go @@ -1,7 +1,9 @@ package utils import ( + "errors" "fmt" + "io/fs" "os" "runtime" ) @@ -41,3 +43,9 @@ func EnsureDirectory(path string, perm os.FileMode) error { // other error opening path return fmt.Errorf("failed to access %s: %w", path, err) } + +// PathExists returns whether the given path (file or dir) exists. +func PathExists(path string) bool { + _, err := os.Stat(path) + return err == nil || errors.Is(err, fs.ErrExist) +} diff --git a/utils/onceagain.go b/utils/onceagain.go index 0d34622..78802b1 100644 --- a/utils/onceagain.go +++ b/utils/onceagain.go @@ -24,7 +24,9 @@ type OnceAgain struct { // Do calls the function f if and only if Do is being called for the // first time for this instance of Once. In other words, given -// var once Once +// +// var once Once +// // if once.Do(f) is called multiple times, only the first call will invoke f, // even if f has a different value in each invocation. A new instance of // Once is required for each function to execute. @@ -32,14 +34,14 @@ type OnceAgain struct { // Do is intended for initialization that must be run exactly once. Since f // is niladic, it may be necessary to use a function literal to capture the // arguments to a function to be invoked by Do: -// config.once.Do(func() { config.init(filename) }) +// +// config.once.Do(func() { config.init(filename) }) // // Because no call to Do returns until the one call to f returns, if f causes // Do to be called, it will deadlock. // // If f panics, Do considers it to have returned; future calls of Do return // without calling f. -// func (o *OnceAgain) Do(f func()) { // Note: Here is an incorrect implementation of Do: // diff --git a/utils/renameio/symlink_test.go b/utils/renameio/symlink_test.go index e8b25c6..a3a1b48 100644 --- a/utils/renameio/symlink_test.go +++ b/utils/renameio/symlink_test.go @@ -4,7 +4,6 @@ package renameio import ( "bytes" - "io/ioutil" "os" "path/filepath" "testing" @@ -13,7 +12,7 @@ import ( func TestSymlink(t *testing.T) { t.Parallel() - d, err := ioutil.TempDir("", "test-renameio-testsymlink") + d, err := os.MkdirTemp("", "test-renameio-testsymlink") if err != nil { t.Fatal(err) } @@ -22,7 +21,7 @@ func TestSymlink(t *testing.T) { }) want := []byte("Hello World") - if err := ioutil.WriteFile(filepath.Join(d, "hello.txt"), want, 0o0600); err != nil { + if err := os.WriteFile(filepath.Join(d, "hello.txt"), want, 0o0600); err != nil { t.Fatal(err) } @@ -31,7 +30,7 @@ func TestSymlink(t *testing.T) { t.Fatal(err) } - got, err := ioutil.ReadFile(filepath.Join(d, "hi.txt")) + got, err := os.ReadFile(filepath.Join(d, "hi.txt")) if err != nil { t.Fatal(err) } diff --git a/utils/renameio/tempfile.go b/utils/renameio/tempfile.go index 8364d64..270bbc9 100644 --- a/utils/renameio/tempfile.go +++ b/utils/renameio/tempfile.go @@ -1,7 +1,6 @@ package renameio import ( - "io/ioutil" "os" "path/filepath" ) @@ -31,7 +30,7 @@ func tempDir(dir, dest string) string { // the TMPDIR environment variable. tmpdir := os.TempDir() - testsrc, err := ioutil.TempFile(tmpdir, "."+filepath.Base(dest)) + testsrc, err := os.CreateTemp(tmpdir, "."+filepath.Base(dest)) if err != nil { return fallback } @@ -43,7 +42,7 @@ func tempDir(dir, dest string) string { }() _ = testsrc.Close() - testdest, err := ioutil.TempFile(filepath.Dir(dest), "."+filepath.Base(dest)) + testdest, err := os.CreateTemp(filepath.Dir(dest), "."+filepath.Base(dest)) if err != nil { return fallback } @@ -114,7 +113,7 @@ func (t *PendingFile) CloseAtomicallyReplace() error { return nil } -// TempFile wraps ioutil.TempFile for the use case of atomically creating or +// TempFile wraps os.CreateTemp for the use case of atomically creating or // replacing the destination file at path. // // If dir is the empty string, TempDir(filepath.Base(path)) is used. If you are @@ -125,7 +124,7 @@ func (t *PendingFile) CloseAtomicallyReplace() error { // The file's permissions will be 0600 by default. You can change these by // explicitly calling Chmod on the returned PendingFile. func TempFile(dir, path string) (*PendingFile, error) { - f, err := ioutil.TempFile(tempDir(dir, path), "."+filepath.Base(path)) + f, err := os.CreateTemp(tempDir(dir, path), "."+filepath.Base(path)) if err != nil { return nil, err } @@ -142,9 +141,9 @@ func Symlink(oldname, newname string) error { return err } - // We need to use ioutil.TempDir, as we cannot overwrite a ioutil.TempFile, + // We need to use os.MkdirTemp, as we cannot overwrite a os.CreateTemp, // and removing+symlinking creates a TOCTOU race. - d, err := ioutil.TempDir(filepath.Dir(newname), "."+filepath.Base(newname)) + d, err := os.MkdirTemp(filepath.Dir(newname), "."+filepath.Base(newname)) if err != nil { return err } diff --git a/utils/renameio/tempfile_linux_test.go b/utils/renameio/tempfile_linux_test.go index 95375e1..88ce025 100644 --- a/utils/renameio/tempfile_linux_test.go +++ b/utils/renameio/tempfile_linux_test.go @@ -3,7 +3,6 @@ package renameio import ( - "io/ioutil" "os" "path/filepath" "syscall" @@ -23,7 +22,7 @@ func TestTempDir(t *testing.T) { }) } - mount1, err := ioutil.TempDir("", "test-renameio-testtempdir1") + mount1, err := os.MkdirTemp("", "test-renameio-testtempdir1") if err != nil { t.Fatal(err) } @@ -31,7 +30,7 @@ func TestTempDir(t *testing.T) { _ = os.RemoveAll(mount1) }) - mount2, err := ioutil.TempDir("", "test-renameio-testtempdir2") + mount2, err := os.MkdirTemp("", "test-renameio-testtempdir2") if err != nil { t.Fatal(err) } diff --git a/utils/renameio/writefile_test.go b/utils/renameio/writefile_test.go index b7d1b49..eaf302b 100644 --- a/utils/renameio/writefile_test.go +++ b/utils/renameio/writefile_test.go @@ -4,7 +4,6 @@ package renameio import ( "bytes" - "io/ioutil" "os" "path/filepath" "testing" @@ -13,7 +12,7 @@ import ( func TestWriteFile(t *testing.T) { t.Parallel() - d, err := ioutil.TempDir("", "test-renameio-testwritefile") + d, err := os.MkdirTemp("", "test-renameio-testwritefile") if err != nil { t.Fatal(err) } @@ -29,7 +28,7 @@ func TestWriteFile(t *testing.T) { t.Fatal(err) } - gotData, err := ioutil.ReadFile(filename) + gotData, err := os.ReadFile(filename) if err != nil { t.Fatal(err) } diff --git a/utils/stablepool.go b/utils/stablepool.go index d253d65..5195b44 100644 --- a/utils/stablepool.go +++ b/utils/stablepool.go @@ -9,7 +9,6 @@ import "sync" // A StablePool is a set of temporary objects that may be individually saved and // retrieved. // -// // In contrast to sync.Pool, items are not removed automatically. Every item // will be returned at some point. Items are returned in a FIFO manner in order // to evenly distribute usage of a set of items. diff --git a/utils/structure_test.go b/utils/structure_test.go index 8cbb2ae..0c277af 100644 --- a/utils/structure_test.go +++ b/utils/structure_test.go @@ -4,7 +4,6 @@ package utils import ( "fmt" - "io/ioutil" "os" "path/filepath" "strings" @@ -23,7 +22,7 @@ func ExampleDirStructure() { // /repo/b/d/f/g/h [707] // /secret [700] - basePath, err := ioutil.TempDir("", "") + basePath, err := os.MkdirTemp("", "") if err != nil { fmt.Println(err) return