Make format and value nullable and improve maintenance and purge queries

base/database/storage/sqlite/migrations/1_initial.sql

@@ -3,8 +3,8 @@
 CREATE TABLE records (
     key TEXT PRIMARY KEY,
 
-    format SMALLINT NOT NULL,
+    format SMALLINT,
-    value  BLOB NOT NULL,
+    value  BLOB,
 
     created    BIGINT NOT NULL,
     modified   BIGINT NOT NULL,

base/database/storage/sqlite/models/records.bob.go

@@ -7,7 +7,9 @@ import (
 	"context"
 	"io"
 
+	"github.com/aarondl/opt/null"
 	"github.com/aarondl/opt/omit"
+	"github.com/aarondl/opt/omitnull"
 	"github.com/stephenafamo/bob"
 	"github.com/stephenafamo/bob/dialect/sqlite"
 	"github.com/stephenafamo/bob/dialect/sqlite/dialect"
@@ -19,15 +21,15 @@ import (
 
 // Record is an object representing the database table.
 type Record struct {
-	Key        string `db:"key,pk" `
+	Key        string           `db:"key,pk" `
-	Format     int16  `db:"format" `
+	Format     null.Val[int16]  `db:"format" `
-	Value      []byte `db:"value" `
+	Value      null.Val[[]byte] `db:"value" `
-	Created    int64  `db:"created" `
+	Created    int64            `db:"created" `
-	Modified   int64  `db:"modified" `
+	Modified   int64            `db:"modified" `
-	Expires    int64  `db:"expires" `
+	Expires    int64            `db:"expires" `
-	Deleted    int64  `db:"deleted" `
+	Deleted    int64            `db:"deleted" `
-	Secret     bool   `db:"secret" `
+	Secret     bool             `db:"secret" `
-	Crownjewel bool   `db:"crownjewel" `
+	Crownjewel bool             `db:"crownjewel" `
 }
 
 // RecordSlice is an alias for a slice of pointers to Record.
@@ -92,8 +94,8 @@ func buildRecordColumns(alias string) recordColumns {
 
 type recordWhere[Q sqlite.Filterable] struct {
 	Key        sqlite.WhereMod[Q, string]
-	Format     sqlite.WhereMod[Q, int16]
+	Format     sqlite.WhereNullMod[Q, int16]
-	Value      sqlite.WhereMod[Q, []byte]
+	Value      sqlite.WhereNullMod[Q, []byte]
 	Created    sqlite.WhereMod[Q, int64]
 	Modified   sqlite.WhereMod[Q, int64]
 	Expires    sqlite.WhereMod[Q, int64]
@@ -109,8 +111,8 @@ func (recordWhere[Q]) AliasedAs(alias string) recordWhere[Q] {
 func buildRecordWhere[Q sqlite.Filterable](cols recordColumns) recordWhere[Q] {
 	return recordWhere[Q]{
 		Key:        sqlite.Where[Q, string](cols.Key),
-		Format:     sqlite.Where[Q, int16](cols.Format),
+		Format:     sqlite.WhereNull[Q, int16](cols.Format),
-		Value:      sqlite.Where[Q, []byte](cols.Value),
+		Value:      sqlite.WhereNull[Q, []byte](cols.Value),
 		Created:    sqlite.Where[Q, int64](cols.Created),
 		Modified:   sqlite.Where[Q, int64](cols.Modified),
 		Expires:    sqlite.Where[Q, int64](cols.Expires),
@@ -124,15 +126,15 @@ func buildRecordWhere[Q sqlite.Filterable](cols recordColumns) recordWhere[Q] {
 // All values are optional, and do not have to be set
 // Generated columns are not included
 type RecordSetter struct {
-	Key        omit.Val[string] `db:"key,pk" `
+	Key        omit.Val[string]     `db:"key,pk" `
-	Format     omit.Val[int16]  `db:"format" `
+	Format     omitnull.Val[int16]  `db:"format" `
-	Value      omit.Val[[]byte] `db:"value" `
+	Value      omitnull.Val[[]byte] `db:"value" `
-	Created    omit.Val[int64]  `db:"created" `
+	Created    omit.Val[int64]      `db:"created" `
-	Modified   omit.Val[int64]  `db:"modified" `
+	Modified   omit.Val[int64]      `db:"modified" `
-	Expires    omit.Val[int64]  `db:"expires" `
+	Expires    omit.Val[int64]      `db:"expires" `
-	Deleted    omit.Val[int64]  `db:"deleted" `
+	Deleted    omit.Val[int64]      `db:"deleted" `
-	Secret     omit.Val[bool]   `db:"secret" `
+	Secret     omit.Val[bool]       `db:"secret" `
-	Crownjewel omit.Val[bool]   `db:"crownjewel" `
+	Crownjewel omit.Val[bool]       `db:"crownjewel" `
 }
 
 func (s RecordSetter) SetColumns() []string {
@@ -181,10 +183,10 @@ func (s RecordSetter) Overwrite(t *Record) {
 		t.Key, _ = s.Key.Get()
 	}
 	if !s.Format.IsUnset() {
-		t.Format, _ = s.Format.Get()
+		t.Format, _ = s.Format.GetNull()
 	}
 	if !s.Value.IsUnset() {
-		t.Value, _ = s.Value.Get()
+		t.Value, _ = s.Value.GetNull()
 	}
 	if !s.Created.IsUnset() {
 		t.Created, _ = s.Created.Get()

base/database/storage/sqlite/sqlite.go

@@ -10,7 +10,15 @@ import (
 	"time"
 
 	"github.com/aarondl/opt/omit"
+	"github.com/aarondl/opt/omitnull"
 	migrate "github.com/rubenv/sql-migrate"
+	sqldblogger "github.com/simukti/sqldb-logger"
+	"github.com/stephenafamo/bob"
+	"github.com/stephenafamo/bob/dialect/sqlite"
+	"github.com/stephenafamo/bob/dialect/sqlite/im"
+	"github.com/stephenafamo/bob/dialect/sqlite/um"
+	_ "modernc.org/sqlite"
+
 	"github.com/safing/portmaster/base/database/accessor"
 	"github.com/safing/portmaster/base/database/iterator"
 	"github.com/safing/portmaster/base/database/query"
@@ -19,12 +27,6 @@ import (
 	"github.com/safing/portmaster/base/database/storage/sqlite/models"
 	"github.com/safing/portmaster/base/log"
 	"github.com/safing/structures/dsd"
-	"github.com/stephenafamo/bob"
-	"github.com/stephenafamo/bob/dialect/sqlite"
-	"github.com/stephenafamo/bob/dialect/sqlite/im"
-	"github.com/stephenafamo/bob/dialect/sqlite/um"
-
-	_ "modernc.org/sqlite"
 )
 
 // SQLite storage.
@@ -47,14 +49,40 @@ func init() {
 
 // NewSQLite creates a sqlite database.
 func NewSQLite(name, location string) (*SQLite, error) {
+	return openSQLite(name, location, false)
+}
+
+// openSQLite creates a sqlite database.
+func openSQLite(name, location string, printStmts bool) (*SQLite, error) {
 	dbFile := filepath.Join(location, "db.sqlite")
 
 	// Open database file.
+	// Default settings:
+	// _time_format = YYYY-MM-DDTHH:MM:SS.SSS
+	// _txlock = deferred
 	db, err := sql.Open("sqlite", dbFile)
 	if err != nil {
 		return nil, fmt.Errorf("open sqlite: %w", err)
 	}
 
+	// Enable statement printing.
+	if printStmts {
+		db = sqldblogger.OpenDriver(dbFile, db.Driver(), &statementLogger{})
+	}
+
+	// Set other settings.
+	pragmas := []string{
+		"PRAGMA journal_mode=WAL;",   // Corruption safe write ahead log for txs.
+		"PRAGMA synchronous=NORMAL;", // Best for WAL.
+		"PRAGMA cache_size=-10000;",  // 10MB Cache.
+	}
+	for _, pragma := range pragmas {
+		_, err := db.Exec(pragma)
+		if err != nil {
+			return nil, fmt.Errorf("failed to init sqlite with %s: %w", pragma, err)
+		}
+	}
+
 	// Run migrations on database.
 	n, err := migrate.Exec(db, "sqlite3", getMigrations(), migrate.Up)
 	if err != nil {
@@ -84,7 +112,13 @@ func (db *SQLite) Get(key string) (record.Record, error) {
 	}
 
 	// Return data in wrapper.
-	return record.NewWrapperFromDatabase(db.name, key, getMeta(r), uint8(r.Format), r.Value)
+	return record.NewWrapperFromDatabase(
+		db.name,
+		key,
+		getMeta(r),
+		uint8(r.Format.GetOrZero()),
+		r.Value.GetOrZero(),
+	)
 }
 
 // GetMeta returns the metadata of a database record.
@@ -114,13 +148,20 @@ func (db *SQLite) putRecord(r record.Record, tx *bob.Tx) (record.Record, error)
 	if err != nil {
 		return nil, err
 	}
+	// Prepare for setter.
+	setFormat := omitnull.From(int16(dsd.JSON))
+	setData := omitnull.From(data)
+	if len(data) == 0 {
+		setFormat.Null()
+		setData.Null()
+	}
 
 	// Create structure for insert.
 	m := r.Meta()
 	setter := models.RecordSetter{
 		Key:        omit.From(r.DatabaseKey()),
-		Format:     omit.From(int16(dsd.JSON)),
+		Format:     setFormat,
-		Value:      omit.From(data),
+		Value:      setData,
 		Created:    omit.From(m.Created),
 		Modified:   omit.From(m.Modified),
 		Expires:    omit.From(m.Expires),
@@ -269,7 +310,11 @@ recordsLoop:
 
 		// Check Data.
 		if q.HasWhereCondition() {
-			jsonData := string(r.Value)
+			if r.Format.IsNull() || r.Value.IsNull() {
+				continue recordsLoop
+			}
+
+			jsonData := string(r.Value.GetOrZero())
 			jsonAccess := accessor.NewJSONAccessor(&jsonData)
 			if !q.MatchesAccessor(jsonAccess) {
 				continue recordsLoop
@@ -277,7 +322,13 @@ recordsLoop:
 		}
 
 		// Build database record.
-		matched, _ := record.NewWrapperFromDatabase(db.name, r.Key, m, uint8(r.Format), r.Value)
+		matched, _ := record.NewWrapperFromDatabase(
+			db.name,
+			r.Key,
+			m,
+			uint8(r.Format.GetOrZero()),
+			r.Value.GetOrZero(),
+		)
 
 		select {
 		case <-queryIter.Done:
@@ -301,7 +352,7 @@ recordsLoop:
 
 // Purge deletes all records that match the given query. It returns the number of successful deletes and an error.
 func (db *SQLite) Purge(ctx context.Context, q *query.Query, local, internal, shadowDelete bool) (int, error) {
-	// Optimize for local and internal queries without where clause.
+	// Optimize for local and internal queries without where clause and without shadow delete.
 	if local && internal && !shadowDelete && !q.HasWhereCondition() {
 		db.lock.Lock()
 		defer db.lock.Unlock()
@@ -321,21 +372,49 @@ func (db *SQLite) Purge(ctx context.Context, q *query.Query, local, internal, sh
 		return int(n), err
 	}
 
-	// Otherwise, iterate over all entries and delete matching ones.
+	// Optimize for local and internal queries without where clause, but with shadow delete.
+	if local && internal && shadowDelete && !q.HasWhereCondition() {
+		db.lock.Lock()
+		defer db.lock.Unlock()
 
-	// Create iterator to check all matching records.
+		// First count entries (SQLite does not support affected rows)
-	queryIter := iterator.New()
+		n, err := models.Records.Query(
-	defer queryIter.Cancel()
+			models.SelectWhere.Records.Key.Like(q.DatabaseKeyPrefix()+"%"),
-	go db.queryExecutor(queryIter, q, local, internal)
+		).Count(db.ctx, db.bob)
+		if err != nil || n == 0 {
+			return int(n), err
+		}
 
-	// Delete all matching records.
+		// Mark purged records as deleted.
-	var deleted int
+		now := time.Now().Unix()
-	for r := range queryIter.Next {
+		_, err = models.Records.Update(
-		db.Delete(r.DatabaseKey())
+			um.SetCol("format").ToArg(nil),
-		deleted++
+			um.SetCol("value").ToArg(nil),
+			um.SetCol("deleted").ToArg(now),
+			models.UpdateWhere.Records.Key.Like(q.DatabaseKeyPrefix()+"%"),
+		).Exec(db.ctx, db.bob)
+		return int(n), err
 	}
 
-	return deleted, nil
+	// Otherwise, iterate over all entries and delete matching ones.
+	return 0, storage.ErrNotImplemented
+
+	// Create iterator to check all matching records.
+
+	// TODO: This is untested and also needs handling of shadowDelete.
+	// For now: Use only without where condition and with a local and internal db interface.
+	// queryIter := iterator.New()
+	// defer queryIter.Cancel()
+	// go db.queryExecutor(queryIter, q, local, internal)
+
+	// // Delete all matching records.
+	// var deleted int
+	// for r := range queryIter.Next {
+	// 	db.Delete(r.DatabaseKey())
+	// 	deleted++
+	// }
+
+	// return deleted, nil
 }
 
 // ReadOnly returns whether the database is read only.
@@ -360,6 +439,8 @@ func (db *SQLite) MaintainRecordStates(ctx context.Context, purgeDeletedBefore t
 	if shadowDelete {
 		// Mark expired records as deleted.
 		models.Records.Update(
+			um.SetCol("format").ToArg(nil),
+			um.SetCol("value").ToArg(nil),
 			um.SetCol("deleted").ToArg(now),
 			models.UpdateWhere.Records.Deleted.EQ(0),
 			models.UpdateWhere.Records.Expires.GT(0),
@@ -416,3 +497,9 @@ func (db *SQLite) Shutdown() error {
 
 	return db.bob.Close()
 }
+
+type statementLogger struct{}
+
+func (sl statementLogger) Log(ctx context.Context, level sqldblogger.Level, msg string, data map[string]interface{}) {
+	fmt.Printf("SQL: %s --- %+v\n", msg, data)
+}

base/database/storage/sqlite/sqlite_test.go

@@ -7,10 +7,11 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/safing/portmaster/base/database/query"
 	"github.com/safing/portmaster/base/database/record"
 	"github.com/safing/portmaster/base/database/storage"
-	"github.com/stretchr/testify/assert"
 )
 
 var (
@@ -51,7 +52,7 @@ func TestSQLite(t *testing.T) {
 	}()
 
 	// start
-	db, err := NewSQLite("test", testDir)
+	db, err := openSQLite("test", testDir, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -105,16 +106,18 @@ func TestSQLite(t *testing.T) {
 	// setup query test records
 	qA := &TestRecord{}
 	qA.SetKey("test:path/to/A")
-	qA.CreateMeta()
+	qA.UpdateMeta()
 	qB := &TestRecord{}
 	qB.SetKey("test:path/to/B")
-	qB.CreateMeta()
+	qB.UpdateMeta()
 	qC := &TestRecord{}
 	qC.SetKey("test:path/to/C")
-	qC.CreateMeta()
+	qC.UpdateMeta()
+	// Set expiry in the past.
+	qC.Meta().Expires = time.Now().Add(-time.Hour).Unix()
 	qZ := &TestRecord{}
 	qZ.SetKey("test:z")
-	qZ.CreateMeta()
+	qZ.UpdateMeta()
 	put, errs := db.PutMany(false)
 	put <- qA
 	put <- qB
@@ -139,7 +142,8 @@ func TestSQLite(t *testing.T) {
 	if it.Err() != nil {
 		t.Fatal(it.Err())
 	}
-	if cnt != 3 {
+	if cnt != 2 {
+		// Note: One is expired.
 		t.Fatalf("unexpected query result count: %d", cnt)
 	}
 
@@ -156,7 +160,7 @@ func TestSQLite(t *testing.T) {
 	}
 
 	// maintenance
-	err = db.MaintainRecordStates(context.TODO(), time.Now(), true)
+	err = db.MaintainRecordStates(context.TODO(), time.Now().Add(-time.Minute), true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -168,11 +172,11 @@ func TestSQLite(t *testing.T) {
 	}
 
 	// purging
-	n, err := db.Purge(context.TODO(), query.New("test:path/to/").MustBeValid(), true, true, false)
+	n, err := db.Purge(context.TODO(), query.New("test:path/to/").MustBeValid(), true, true, true)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if n != 3 {
+	if n != 2 {
 		t.Fatalf("unexpected purge delete count: %d", n)
 	}
 

go.mod

@@ -145,6 +145,7 @@ require (
 	github.com/satori/go.uuid v1.2.0 // indirect
 	github.com/seehuhn/sha256d v1.0.0 // indirect
 	github.com/shopspring/decimal v1.3.1 // indirect
+	github.com/simukti/sqldb-logger v0.0.0-20230108155151-646c1a075551 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stephenafamo/scan v0.6.1 // indirect

go.sum

@@ -367,6 +367,8 @@ github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMT
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/simukti/sqldb-logger v0.0.0-20230108155151-646c1a075551 h1:+EXKKt7RC4HyE/iE8zSeFL+7YBL8Z7vpBaEE3c7lCnk=
+github.com/simukti/sqldb-logger v0.0.0-20230108155151-646c1a075551/go.mod h1:ztTX0ctjRZ1wn9OXrzhonvNmv43yjFUXJYJR95JQAJE=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v0.0.0-20170901052352-ee1bd8ee15a1/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=