From 099a597be9a5289ad5e68009e37af56eb4af196d Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 24 Sep 2019 15:38:10 +0200 Subject: [PATCH 1/5] Move crypto/random to rng --- crypto/random/doc.go | 7 ------- rng/doc.go | 9 +++++++++ {crypto/random => rng}/entropy.go | 2 +- {crypto/random => rng}/entropy_test.go | 2 +- {crypto/random => rng}/fullfeed.go | 2 +- {crypto/random => rng}/fullfeed_test.go | 2 +- {crypto/random => rng}/get.go | 2 +- {crypto/random => rng}/get_test.go | 2 +- {crypto/random => rng}/osfeeder.go | 2 +- {crypto/random => rng}/rng.go | 2 +- {crypto/random => rng}/rng_test.go | 2 +- {crypto/random => rng}/test/.gitignore | 0 {crypto/random => rng}/test/README.md | 0 {crypto/random => rng}/test/main.go | 6 +++--- {crypto/random => rng}/tickfeeder.go | 2 +- 15 files changed, 22 insertions(+), 20 deletions(-) delete mode 100644 crypto/random/doc.go create mode 100644 rng/doc.go rename {crypto/random => rng}/entropy.go (99%) rename {crypto/random => rng}/entropy_test.go (98%) rename {crypto/random => rng}/fullfeed.go (97%) rename {crypto/random => rng}/fullfeed_test.go (90%) rename {crypto/random => rng}/get.go (99%) rename {crypto/random => rng}/get_test.go (98%) rename {crypto/random => rng}/osfeeder.go (97%) rename {crypto/random => rng}/rng.go (99%) rename {crypto/random => rng}/rng_test.go (98%) rename {crypto/random => rng}/test/.gitignore (100%) rename {crypto/random => rng}/test/README.md (100%) rename {crypto/random => rng}/test/main.go (95%) rename {crypto/random => rng}/tickfeeder.go (98%) diff --git a/crypto/random/doc.go b/crypto/random/doc.go deleted file mode 100644 index b1fa573..0000000 --- a/crypto/random/doc.go +++ /dev/null @@ -1,7 +0,0 @@ -// Package random provides a feedable CSPRNG. -// -// CSPRNG used is fortuna: github.com/seehuhn/fortuna -// By default the CSPRNG is fed by two sources: -// - OS RNG -// - Entropy gathered by context switching -package random diff --git a/rng/doc.go b/rng/doc.go new file mode 100644 index 0000000..b7fed11 --- /dev/null +++ b/rng/doc.go @@ -0,0 +1,9 @@ +// Package rng provides a feedable CSPRNG. +// +// CSPRNG used is fortuna: github.com/seehuhn/fortuna +// By default the CSPRNG is fed by two sources: +// - It starts with a seed from `crypto/rand` and periodically reseeds from there +// - A really simple tickfeeder which extracts entropy from the internal go scheduler using goroutines and is meant to be used under load. +// +// The RNG can also be easily fed with additional sources. +package rng diff --git a/crypto/random/entropy.go b/rng/entropy.go similarity index 99% rename from crypto/random/entropy.go rename to rng/entropy.go index 96dea36..b85ff88 100644 --- a/crypto/random/entropy.go +++ b/rng/entropy.go @@ -1,4 +1,4 @@ -package random +package rng import ( "encoding/binary" diff --git a/crypto/random/entropy_test.go b/rng/entropy_test.go similarity index 98% rename from crypto/random/entropy_test.go rename to rng/entropy_test.go index 3418910..08a2d62 100644 --- a/crypto/random/entropy_test.go +++ b/rng/entropy_test.go @@ -1,4 +1,4 @@ -package random +package rng import ( "testing" diff --git a/crypto/random/fullfeed.go b/rng/fullfeed.go similarity index 97% rename from crypto/random/fullfeed.go rename to rng/fullfeed.go index 8a99ed6..93f2ddb 100644 --- a/crypto/random/fullfeed.go +++ b/rng/fullfeed.go @@ -1,4 +1,4 @@ -package random +package rng import ( "time" diff --git a/crypto/random/fullfeed_test.go b/rng/fullfeed_test.go similarity index 90% rename from crypto/random/fullfeed_test.go rename to rng/fullfeed_test.go index bc32d40..5e8b84c 100644 --- a/crypto/random/fullfeed_test.go +++ b/rng/fullfeed_test.go @@ -1,4 +1,4 @@ -package random +package rng import ( "testing" diff --git a/crypto/random/get.go b/rng/get.go similarity index 99% rename from crypto/random/get.go rename to rng/get.go index efe6cb6..eb94130 100644 --- a/crypto/random/get.go +++ b/rng/get.go @@ -1,4 +1,4 @@ -package random +package rng import ( "encoding/binary" diff --git a/crypto/random/get_test.go b/rng/get_test.go similarity index 98% rename from crypto/random/get_test.go rename to rng/get_test.go index 2bcc3dd..38444ec 100644 --- a/crypto/random/get_test.go +++ b/rng/get_test.go @@ -1,4 +1,4 @@ -package random +package rng import ( "testing" diff --git a/crypto/random/osfeeder.go b/rng/osfeeder.go similarity index 97% rename from crypto/random/osfeeder.go rename to rng/osfeeder.go index 349f86a..7d1bc5f 100644 --- a/crypto/random/osfeeder.go +++ b/rng/osfeeder.go @@ -1,4 +1,4 @@ -package random +package rng import ( "crypto/rand" diff --git a/crypto/random/rng.go b/rng/rng.go similarity index 99% rename from crypto/random/rng.go rename to rng/rng.go index 09be8ba..85528f1 100644 --- a/crypto/random/rng.go +++ b/rng/rng.go @@ -1,4 +1,4 @@ -package random +package rng import ( "crypto/aes" diff --git a/crypto/random/rng_test.go b/rng/rng_test.go similarity index 98% rename from crypto/random/rng_test.go rename to rng/rng_test.go index 555f5c7..ea7708a 100644 --- a/crypto/random/rng_test.go +++ b/rng/rng_test.go @@ -1,4 +1,4 @@ -package random +package rng import ( "testing" diff --git a/crypto/random/test/.gitignore b/rng/test/.gitignore similarity index 100% rename from crypto/random/test/.gitignore rename to rng/test/.gitignore diff --git a/crypto/random/test/README.md b/rng/test/README.md similarity index 100% rename from crypto/random/test/README.md rename to rng/test/README.md diff --git a/crypto/random/test/main.go b/rng/test/main.go similarity index 95% rename from crypto/random/test/main.go rename to rng/test/main.go index 274a210..601ac95 100644 --- a/crypto/random/test/main.go +++ b/rng/test/main.go @@ -12,7 +12,7 @@ import ( "runtime" "time" - "github.com/safing/portbase/crypto/random" + "github.com/safing/portbase/rng" ) func noise() { @@ -55,13 +55,13 @@ func main() { switch os.Args[1] { case "fortuna": - err := random.Start() + err := rng.Start() if err != nil { panic(err) } for { - b, err := random.Bytes(64) + b, err := rng.Bytes(64) if err != nil { panic(err) } diff --git a/crypto/random/tickfeeder.go b/rng/tickfeeder.go similarity index 98% rename from crypto/random/tickfeeder.go rename to rng/tickfeeder.go index 8eb7f8f..45765ee 100644 --- a/crypto/random/tickfeeder.go +++ b/rng/tickfeeder.go @@ -1,4 +1,4 @@ -package random +package rng import ( "time" From d0ffecb06035365d623e39b0c10054a40d5787d8 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 24 Sep 2019 15:38:23 +0200 Subject: [PATCH 2/5] Improve test output readability --- test | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test b/test index 67fdac5..003aa41 100755 --- a/test +++ b/test @@ -174,13 +174,15 @@ echo "running tests for ${platformInfo//$'\n'/ }:" # run vet/test on packages for package in $packages; do + echo "" + echo $package checkformat $package run golint -set_exit_status -min_confidence 1.0 $package run go vet $package - run go test -cover $fullTestFlags $package if [[ $all -eq 1 ]]; then run golangci-lint run $GOPATH/src/$package fi + run go test -cover $fullTestFlags $package done echo "" From f7ccc58b9899b4098c96b4baa6c7fa3183be262c Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 24 Sep 2019 15:38:51 +0200 Subject: [PATCH 3/5] Update rng package import --- api/authentication.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/authentication.go b/api/authentication.go index 80413e9..baeba1d 100644 --- a/api/authentication.go +++ b/api/authentication.go @@ -6,8 +6,8 @@ import ( "sync" "time" - "github.com/safing/portbase/crypto/random" "github.com/safing/portbase/log" + "github.com/safing/portbase/rng" ) var ( @@ -91,7 +91,7 @@ func authMiddleware(next http.Handler) http.Handler { } // write new cookie - token, err := random.Bytes(32) // 256 bit + token, err := rng.Bytes(32) // 256 bit if err != nil { log.Warningf("api: failed to generate random token: %s", err) http.Error(w, "Internal Server Error", http.StatusInternalServerError) From 32e8c555f59c53f8c23d3c7a609852e0e9fe739e Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 24 Sep 2019 15:39:10 +0200 Subject: [PATCH 4/5] Fix lint issue --- container/container.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/container/container.go b/container/container.go index ebad1ab..2532a40 100644 --- a/container/container.go +++ b/container/container.go @@ -173,10 +173,7 @@ func (c *Container) CheckError() { // HasError returns wether or not the container is holding an error. func (c *Container) HasError() bool { - if c.err != nil { - return true - } - return false + return c.err != nil } // Error returns the error. From e4d8915007b6adbf9ae57eda0549572eb18c21a6 Mon Sep 17 00:00:00 2001 From: Daniel Date: Tue, 24 Sep 2019 15:39:19 +0200 Subject: [PATCH 5/5] Update README --- README.md | 51 ++++++++++++++++++++++----------------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index 877516c..f8e9fa0 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,13 @@ Portbase helps you quickly take off with your project. It gives you all the basi Here is what is included: - `log`: really fast and beautiful logging -- `modules`: a multi stage, dependency aware boot process for your software +- `modules`: a multi stage, dependency aware boot process for your software, also manages tasks - `config`: simple, live updating and extremely fast configuration storage - `info`: easily tag your builds with versions, commit hashes, and so on -- `taskmanager`: run your more important goroutines first - `formats`: some handy data encoding libs -- `crypto/hash`: easy self-identifying hashes -- `crypto/random`: a feedable CSPRNG for great randomness +- `rng`: a feedable CSPRNG for great randomness - `database`: intelligent and syncable database with hooks and easy integration with structs, uses buckets with different backends -- `api`: a RESTful and GraphQL hybrid interface to the database +- `api`: a websocket interface to the database, can be extended with custom http handlers Before you continue, a word about this project. It was created to hold the base code for both Portmaster and Gate17. This is also what it will be developed for. If you have a great idea on how to improve portbase, please, by all means, raise an issue and tell us about it, but please also don't be surprised or offended if we ask you to create your own fork to do what you need. Portbase isn't for everyone, it's quite specific to our needs, but we decided to make it easily available to others. @@ -34,22 +32,24 @@ Registering only requires a name/key and the `prep()`, `start()` and `stop()` fu This is how modules are booted: -- `init()` available: ~~flags~~, ~~logging~~, ~~dependencies~~ +- `init()` available: ~~flags~~, ~~config~~, ~~logging~~, ~~dependencies~~ - register flags (with the stdlib `flag` library) - - register config variables - register module -- `module.prep()` available: flags, ~~logging~~, ~~dependencies~~ +- `module.prep()` available: flags, ~~config~~, ~~logging~~, ~~dependencies~~ - react to flags + - register config variables - if an error occurs, return it - return ErrCleanExit for a clean, successful exit. (eg. you only printed a version) -- `module.start()` available: flags, logging, dependencies - - start actual work (ie. goroutines) +- `module.start()` available: flags, config, logging, dependencies + - start tasks and workers - do not log errors while starting, but return them -- `module.stop()` available: flags, logging, dependencies +- `module.stop()` available: flags, config, logging, dependencies - stop all work (ie. goroutines) - do not log errors while stopping, but return them -## config +You can start tasks and workers from your module that are then integrated into the module system and will allow for insights and better control of them in the future. + +## config requires `log` The config package stores the configuration in json strings. This may sound a bit weird, but it's very practical. @@ -59,13 +59,13 @@ When using config variables, you get a function that checks if your config varia // This is how you would get a string config variable function. myVar := GetAsString("my_config_var", "default") - // You then use myVar() directly every time, except you must guarantee the same value between two calls + // You then use myVar() directly every time, except when you must guarantee the same value between two calls if myVar() != "default" { log.Infof("my_config_var is set to %s", myVar()) } // no error handling needed! :) -WARNING: While these config variable functions are _extremely_ fast, they are _NOT_ thread/goroutine safe! +WARNING: While these config variable functions are _extremely_ fast, they are _NOT_ thread/goroutine safe! (Use the `Concurrent` wrapper for that!) ## info @@ -73,9 +73,9 @@ Info provides a easy way to store your version and build information within the The `build` script extracts information from the host and the git repo and then calls `go build` with some additional arguments. -## taskmanager +## formats/varint -The taskmanager lets prioritize goroutines in order to optimize efficiency of your program. The idea is to hold back non time-critical goroutines for periods where no important goroutines are running. +This is just a convenience wrapper around `encoding/binary`, because we use varints a lot. ## formats/dsd requires `formats/varint` @@ -86,27 +86,20 @@ DSD stands for dynamically structured data. In short, this a generic packer that This makes it easier / more efficient to store different data types in a k/v data storage. -## formats/varint +## rng requires `log`, `config` -This is just a convenience wrapper around `encoding/binary`, because we use varints a lot. - -## crypto/hash -_introduction to be written_ - -## crypto/random - -This packege provides a CSPRNG based on the [Fortuna](https://en.wikipedia.org/wiki/Fortuna_(PRNG) CSPRNG, devised by Bruce Schneier and Niels Ferguson. Implemented by Jochen Voss, published [on Github](https://github.com/seehuhn/fortuna). +This package provides a CSPRNG based on the [Fortuna](https://en.wikipedia.org/wiki/Fortuna_(PRNG)) CSPRNG, devised by Bruce Schneier and Niels Ferguson. Implemented by Jochen Voss, published [on Github](https://github.com/seehuhn/fortuna). Only the Generator is used from the `fortuna` package. The feeding system implemented here is configurable and is focused with efficiency in mind. While you can feed the RNG yourself, it has two feeders by default: -- It starts with a seed from `crypt/rand` and periodically reseeds from there -- A really simple tickfeeder which pools the least significant bit of `time.Now().UnixNano()` every time it _ticks_ and feeds to the RNG when it reaches the needed entropy. +- It starts with a seed from `crypto/rand` and periodically reseeds from there +- A really simple tickfeeder which extracts entropy from the internal go scheduler using goroutines and is meant to be used under load. -## database +## database requires `log` _introduction to be written_ -## api requires `database` +## api requires `log`, `database`, `config` _introduction to be written_ ## The main program