vikarti.anatra/safing-portmaster
1
0
Fork 0
mirror of https://github.com/safing/portmaster synced 2025-04-03 10:39:13 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make format and value nullable and improve maintenance and purge queries

Browse source
This commit is contained in:
Daniel 2025-02-28 10:21:59 +01:00
parent b68646c689
commit c04213219b
6 changed files with 155 additions and 59 deletions
base/database/storage/sqlite
migrations
1_initial.sql
models
records.bob.go
sqlite.gosqlite_test.go
go.modgo.sum

4
base/database/storage/sqlite/migrations/1_initial.sql
View file

@ -3,8 +3,8 @@
CREATE TABLE records (
key TEXT PRIMARY KEY,
format SMALLINT NOT NULL,
value BLOB NOT NULL,
format SMALLINT,
value BLOB,
created BIGINT NOT NULL,
modified BIGINT NOT NULL,

50
base/database/storage/sqlite/models/records.bob.go
View file

@ -7,7 +7,9 @@ import (
"context"
"io"
"github.com/aarondl/opt/null"
"github.com/aarondl/opt/omit"
"github.com/aarondl/opt/omitnull"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/sqlite"
"github.com/stephenafamo/bob/dialect/sqlite/dialect"
@ -19,15 +21,15 @@ import (
// Record is an object representing the database table.
type Record struct {
Key string `db:"key,pk" `
Format int16 `db:"format" `
Value []byte `db:"value" `
Created int64 `db:"created" `
Modified int64 `db:"modified" `
Expires int64 `db:"expires" `
Deleted int64 `db:"deleted" `
Secret bool `db:"secret" `
Crownjewel bool `db:"crownjewel" `
Key string `db:"key,pk" `
Format null.Val[int16] `db:"format" `
Value null.Val[[]byte] `db:"value" `
Created int64 `db:"created" `
Modified int64 `db:"modified" `
Expires int64 `db:"expires" `
Deleted int64 `db:"deleted" `
Secret bool `db:"secret" `
Crownjewel bool `db:"crownjewel" `
}
// RecordSlice is an alias for a slice of pointers to Record.
@ -92,8 +94,8 @@ func buildRecordColumns(alias string) recordColumns {
type recordWhere[Q sqlite.Filterable] struct {
Key sqlite.WhereMod[Q, string]
Format sqlite.WhereMod[Q, int16]
Value sqlite.WhereMod[Q, []byte]
Format sqlite.WhereNullMod[Q, int16]
Value sqlite.WhereNullMod[Q, []byte]
Created sqlite.WhereMod[Q, int64]
Modified sqlite.WhereMod[Q, int64]
Expires sqlite.WhereMod[Q, int64]
@ -109,8 +111,8 @@ func (recordWhere[Q]) AliasedAs(alias string) recordWhere[Q] {
func buildRecordWhere[Q sqlite.Filterable](cols recordColumns) recordWhere[Q] {
return recordWhere[Q]{
Key: sqlite.Where[Q, string](cols.Key),
Format: sqlite.Where[Q, int16](cols.Format),
Value: sqlite.Where[Q, []byte](cols.Value),
Format: sqlite.WhereNull[Q, int16](cols.Format),
Value: sqlite.WhereNull[Q, []byte](cols.Value),
Created: sqlite.Where[Q, int64](cols.Created),
Modified: sqlite.Where[Q, int64](cols.Modified),
Expires: sqlite.Where[Q, int64](cols.Expires),
@ -124,15 +126,15 @@ func buildRecordWhere[Q sqlite.Filterable](cols recordColumns) recordWhere[Q] {
// All values are optional, and do not have to be set
// Generated columns are not included
type RecordSetter struct {
Key omit.Val[string] `db:"key,pk" `
Format omit.Val[int16] `db:"format" `
Value omit.Val[[]byte] `db:"value" `
Created omit.Val[int64] `db:"created" `
Modified omit.Val[int64] `db:"modified" `
Expires omit.Val[int64] `db:"expires" `
Deleted omit.Val[int64] `db:"deleted" `
Secret omit.Val[bool] `db:"secret" `
Crownjewel omit.Val[bool] `db:"crownjewel" `
Key omit.Val[string] `db:"key,pk" `
Format omitnull.Val[int16] `db:"format" `
Value omitnull.Val[[]byte] `db:"value" `
Created omit.Val[int64] `db:"created" `
Modified omit.Val[int64] `db:"modified" `
Expires omit.Val[int64] `db:"expires" `
Deleted omit.Val[int64] `db:"deleted" `
Secret omit.Val[bool] `db:"secret" `
Crownjewel omit.Val[bool] `db:"crownjewel" `
}
func (s RecordSetter) SetColumns() []string {
@ -181,10 +183,10 @@ func (s RecordSetter) Overwrite(t *Record) {
t.Key, _ = s.Key.Get()
}
if !s.Format.IsUnset() {
t.Format, _ = s.Format.Get()
t.Format, _ = s.Format.GetNull()
}
if !s.Value.IsUnset() {
t.Value, _ = s.Value.Get()
t.Value, _ = s.Value.GetNull()
}
if !s.Created.IsUnset() {
t.Created, _ = s.Created.Get()

133
base/database/storage/sqlite/sqlite.go
View file

@ -10,7 +10,15 @@ import (
"time"
"github.com/aarondl/opt/omit"
"github.com/aarondl/opt/omitnull"
migrate "github.com/rubenv/sql-migrate"
sqldblogger "github.com/simukti/sqldb-logger"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/sqlite"
"github.com/stephenafamo/bob/dialect/sqlite/im"
"github.com/stephenafamo/bob/dialect/sqlite/um"
_ "modernc.org/sqlite"
"github.com/safing/portmaster/base/database/accessor"
"github.com/safing/portmaster/base/database/iterator"
"github.com/safing/portmaster/base/database/query"
@ -19,12 +27,6 @@ import (
"github.com/safing/portmaster/base/database/storage/sqlite/models"
"github.com/safing/portmaster/base/log"
"github.com/safing/structures/dsd"
"github.com/stephenafamo/bob"
"github.com/stephenafamo/bob/dialect/sqlite"
"github.com/stephenafamo/bob/dialect/sqlite/im"
"github.com/stephenafamo/bob/dialect/sqlite/um"
_ "modernc.org/sqlite"
)
// SQLite storage.
@ -47,14 +49,40 @@ func init() {
// NewSQLite creates a sqlite database.
func NewSQLite(name, location string) (*SQLite, error) {
return openSQLite(name, location, false)
}
// openSQLite creates a sqlite database.
func openSQLite(name, location string, printStmts bool) (*SQLite, error) {
dbFile := filepath.Join(location, "db.sqlite")
// Open database file.
// Default settings:
// _time_format = YYYY-MM-DDTHH:MM:SS.SSS
// _txlock = deferred
db, err := sql.Open("sqlite", dbFile)
if err != nil {
return nil, fmt.Errorf("open sqlite: %w", err)
}
// Enable statement printing.
if printStmts {
db = sqldblogger.OpenDriver(dbFile, db.Driver(), &statementLogger{})
}
// Set other settings.
pragmas := []string{
"PRAGMA journal_mode=WAL;", // Corruption safe write ahead log for txs.
"PRAGMA synchronous=NORMAL;", // Best for WAL.
"PRAGMA cache_size=-10000;", // 10MB Cache.
}
for _, pragma := range pragmas {
_, err := db.Exec(pragma)
if err != nil {
return nil, fmt.Errorf("failed to init sqlite with %s: %w", pragma, err)
}
}
// Run migrations on database.
n, err := migrate.Exec(db, "sqlite3", getMigrations(), migrate.Up)
if err != nil {
@ -84,7 +112,13 @@ func (db *SQLite) Get(key string) (record.Record, error) {
}
// Return data in wrapper.
return record.NewWrapperFromDatabase(db.name, key, getMeta(r), uint8(r.Format), r.Value)
return record.NewWrapperFromDatabase(
db.name,
key,
getMeta(r),
uint8(r.Format.GetOrZero()),
r.Value.GetOrZero(),
)
}
// GetMeta returns the metadata of a database record.
@ -114,13 +148,20 @@ func (db *SQLite) putRecord(r record.Record, tx *bob.Tx) (record.Record, error)
if err != nil {
return nil, err
}
// Prepare for setter.
setFormat := omitnull.From(int16(dsd.JSON))
setData := omitnull.From(data)
if len(data) == 0 {
setFormat.Null()
setData.Null()
}
// Create structure for insert.
m := r.Meta()
setter := models.RecordSetter{
Key: omit.From(r.DatabaseKey()),
Format: omit.From(int16(dsd.JSON)),
Value: omit.From(data),
Format: setFormat,
Value: setData,
Created: omit.From(m.Created),
Modified: omit.From(m.Modified),
Expires: omit.From(m.Expires),
@ -269,7 +310,11 @@ recordsLoop:
// Check Data.
if q.HasWhereCondition() {
jsonData := string(r.Value)
if r.Format.IsNull() || r.Value.IsNull() {
continue recordsLoop
}
jsonData := string(r.Value.GetOrZero())
jsonAccess := accessor.NewJSONAccessor(&jsonData)
if !q.MatchesAccessor(jsonAccess) {
continue recordsLoop
@ -277,7 +322,13 @@ recordsLoop:
}
// Build database record.
matched, _ := record.NewWrapperFromDatabase(db.name, r.Key, m, uint8(r.Format), r.Value)
matched, _ := record.NewWrapperFromDatabase(
db.name,
r.Key,
m,
uint8(r.Format.GetOrZero()),
r.Value.GetOrZero(),
)
select {
case <-queryIter.Done:
@ -301,7 +352,7 @@ recordsLoop:
// Purge deletes all records that match the given query. It returns the number of successful deletes and an error.
func (db *SQLite) Purge(ctx context.Context, q *query.Query, local, internal, shadowDelete bool) (int, error) {
// Optimize for local and internal queries without where clause.
// Optimize for local and internal queries without where clause and without shadow delete.
if local && internal && !shadowDelete && !q.HasWhereCondition() {
db.lock.Lock()
defer db.lock.Unlock()
@ -321,21 +372,49 @@ func (db *SQLite) Purge(ctx context.Context, q *query.Query, local, internal, sh
return int(n), err
}
// Otherwise, iterate over all entries and delete matching ones.
// Optimize for local and internal queries without where clause, but with shadow delete.
if local && internal && shadowDelete && !q.HasWhereCondition() {
db.lock.Lock()
defer db.lock.Unlock()
// Create iterator to check all matching records.
queryIter := iterator.New()
defer queryIter.Cancel()
go db.queryExecutor(queryIter, q, local, internal)
// First count entries (SQLite does not support affected rows)
n, err := models.Records.Query(
models.SelectWhere.Records.Key.Like(q.DatabaseKeyPrefix()+"%"),
).Count(db.ctx, db.bob)
if err != nil || n == 0 {
return int(n), err
}
// Delete all matching records.
var deleted int
for r := range queryIter.Next {
db.Delete(r.DatabaseKey())
deleted++
// Mark purged records as deleted.
now := time.Now().Unix()
_, err = models.Records.Update(
um.SetCol("format").ToArg(nil),
um.SetCol("value").ToArg(nil),
um.SetCol("deleted").ToArg(now),
models.UpdateWhere.Records.Key.Like(q.DatabaseKeyPrefix()+"%"),
).Exec(db.ctx, db.bob)
return int(n), err
}
return deleted, nil
// Otherwise, iterate over all entries and delete matching ones.
return 0, storage.ErrNotImplemented
// Create iterator to check all matching records.
// TODO: This is untested and also needs handling of shadowDelete.
// For now: Use only without where condition and with a local and internal db interface.
// queryIter := iterator.New()
// defer queryIter.Cancel()
// go db.queryExecutor(queryIter, q, local, internal)
// // Delete all matching records.
// var deleted int
// for r := range queryIter.Next {
// db.Delete(r.DatabaseKey())
// deleted++
// }
// return deleted, nil
}
// ReadOnly returns whether the database is read only.
@ -360,6 +439,8 @@ func (db *SQLite) MaintainRecordStates(ctx context.Context, purgeDeletedBefore t
if shadowDelete {
// Mark expired records as deleted.
models.Records.Update(
um.SetCol("format").ToArg(nil),
um.SetCol("value").ToArg(nil),
um.SetCol("deleted").ToArg(now),
models.UpdateWhere.Records.Deleted.EQ(0),
models.UpdateWhere.Records.Expires.GT(0),
@ -416,3 +497,9 @@ func (db *SQLite) Shutdown() error {
return db.bob.Close()
}
type statementLogger struct{}
func (sl statementLogger) Log(ctx context.Context, level sqldblogger.Level, msg string, data map[string]interface{}) {
fmt.Printf("SQL: %s --- %+v\n", msg, data)
}

24
base/database/storage/sqlite/sqlite_test.go
View file

@ -7,10 +7,11 @@ import (
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/safing/portmaster/base/database/query"
"github.com/safing/portmaster/base/database/record"
"github.com/safing/portmaster/base/database/storage"
"github.com/stretchr/testify/assert"
)
var (
@ -51,7 +52,7 @@ func TestSQLite(t *testing.T) {
}()
// start
db, err := NewSQLite("test", testDir)
db, err := openSQLite("test", testDir, true)
if err != nil {
t.Fatal(err)
}
@ -105,16 +106,18 @@ func TestSQLite(t *testing.T) {
// setup query test records
qA := &TestRecord{}
qA.SetKey("test:path/to/A")
qA.CreateMeta()
qA.UpdateMeta()
qB := &TestRecord{}
qB.SetKey("test:path/to/B")
qB.CreateMeta()
qB.UpdateMeta()
qC := &TestRecord{}
qC.SetKey("test:path/to/C")
qC.CreateMeta()
qC.UpdateMeta()
// Set expiry in the past.
qC.Meta().Expires = time.Now().Add(-time.Hour).Unix()
qZ := &TestRecord{}
qZ.SetKey("test:z")
qZ.CreateMeta()
qZ.UpdateMeta()
put, errs := db.PutMany(false)
put <- qA
put <- qB
@ -139,7 +142,8 @@ func TestSQLite(t *testing.T) {
if it.Err() != nil {
t.Fatal(it.Err())
}
if cnt != 3 {
if cnt != 2 {
// Note: One is expired.
t.Fatalf("unexpected query result count: %d", cnt)
}
@ -156,7 +160,7 @@ func TestSQLite(t *testing.T) {
}
// maintenance
err = db.MaintainRecordStates(context.TODO(), time.Now(), true)
err = db.MaintainRecordStates(context.TODO(), time.Now().Add(-time.Minute), true)
if err != nil {
t.Fatal(err)
}
@ -168,11 +172,11 @@ func TestSQLite(t *testing.T) {
}
// purging
n, err := db.Purge(context.TODO(), query.New("test:path/to/").MustBeValid(), true, true, false)
n, err := db.Purge(context.TODO(), query.New("test:path/to/").MustBeValid(), true, true, true)
if err != nil {
t.Fatal(err)
}
if n != 3 {
if n != 2 {
t.Fatalf("unexpected purge delete count: %d", n)
}

1
go.mod
View file

@ -145,6 +145,7 @@ require (
github.com/satori/go.uuid v1.2.0 // indirect
github.com/seehuhn/sha256d v1.0.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
github.com/simukti/sqldb-logger v0.0.0-20230108155151-646c1a075551 // indirect
github.com/spf13/cast v1.5.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stephenafamo/scan v0.6.1 // indirect

2
go.sum
View file

@ -367,6 +367,8 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT
github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/simukti/sqldb-logger v0.0.0-20230108155151-646c1a075551 h1:+EXKKt7RC4HyE/iE8zSeFL+7YBL8Z7vpBaEE3c7lCnk=
github.com/simukti/sqldb-logger v0.0.0-20230108155151-646c1a075551/go.mod h1:ztTX0ctjRZ1wn9OXrzhonvNmv43yjFUXJYJR95JQAJE=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=