vikarti.anatra/safing-portbase
1
0
Fork 0
mirror of https://github.com/safing/portbase synced 2025-09-01 10:09:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Finish query package for now

Browse source
This commit is contained in:
Daniel 2018-08-31 17:11:59 +02:00
parent 115b18dfb6
commit e40d66e103
15 changed files with 714 additions and 181 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

55
database/query/README.md Normal file
View file

@ -0,0 +1,55 @@
# Query
## Control Flow
- Grouping with `(` and `)`
- Chaining with `and` and `or`
- _NO_ mixing! Be explicit and use grouping.
- Negation with `not`
- in front of expression for group: `not (...)`
- inside expression for clause: `name not matches "^King "`
## Selectors
Supported by all feeders:
- root level field: `field`
- sub level field: `field.sub`
- array/slice/map access: `map.0`
- array/slice/map length: `map.#`
Please note that some feeders may have other special characters. It is advised to only use alphanumeric characters for keys.
## Operators
| Name | Textual | Req. Type | Internal Type | Compared with |
|---|---|---|---|
| Equals | `==` | int | int64 | `==` |
| GreaterThan | `>` | int | int64 | `>` |
| GreaterThanOrEqual | `>=` | int | int64 | `>=` |
| LessThan | `<` | int | int64 | `<` |
| LessThanOrEqual | `<=` | int | int64 | `<=` |
| FloatEquals | `f==` | float | float64 | `==` |
| FloatGreaterThan | `f>` | float | float64 | `>` |
| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` |
| FloatLessThan | `f<` | float | float64 | `<` |
| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` |
| SameAs | `sameas`, `s==` | string | string | `==` |
| Contains | `contains`, `co` | string | string | `strings.Contains()` |
| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` |
| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` |
| In | `in` | string | string | for loop with `==` |
| Matches | `matches`, `re` | string | int64 | `regexp.Regexp.Matches()` |
| Is | `is` | bool* | bool | `==` |
| Exists | `exists`, `ex` | any | n/a | n/a |
\*accepts strings: 1, t, T, TRUE, true, True, 0, f, F, FALSE
## Escaping
If you need to use a control character within a value (ie. not for controlling), escape it with `\`.
It is recommended to wrap a word into parenthesis instead of escaping control characters, when possible.
| Location | Characters to be escaped |
|---|---|
| Within parenthesis (`"`) | `"`, `\` |
| Everywhere else | `(`, `)`, `"`, `\`, `\t`, `\r`, `\n`, ` ` (space) |

2
database/query/condition-bool.go
View file

@ -64,5 +64,5 @@ func (c *boolCondition) check() error {
}
func (c *boolCondition) string() string {
return fmt.Sprintf("%s %s %t", c.key, getOpName(c.operator), c.value)
return fmt.Sprintf("%s %s %t", escapeString(c.key), getOpName(c.operator), c.value)
}

2
database/query/condition-exists.go
View file

@ -29,5 +29,5 @@ func (c *existsCondition) check() error {
}
func (c *existsCondition) string() string {
return fmt.Sprintf("%s %s", c.key, getOpName(c.operator))
return fmt.Sprintf("%s %s", escapeString(c.key), getOpName(c.operator))
}

4
database/query/condition-float.go
View file

@ -31,6 +31,8 @@ func newFloatCondition(key string, operator uint8, value interface{}) *floatCond
parsedValue = float64(v)
case uint8:
parsedValue = float64(v)
case uint16:
parsedValue = float64(v)
case uint32:
parsedValue = float64(v)
case float32:
@ -90,5 +92,5 @@ func (c *floatCondition) check() error {
}
func (c *floatCondition) string() string {
return fmt.Sprintf("%s %s %f", c.key, getOpName(c.operator), c.value)
return fmt.Sprintf("%s %s %g", escapeString(c.key), getOpName(c.operator), c.value)
}

4
database/query/condition-int.go
View file

@ -31,6 +31,8 @@ func newIntCondition(key string, operator uint8, value interface{}) *intConditio
parsedValue = int64(v)
case uint8:
parsedValue = int64(v)
case uint16:
parsedValue = int64(v)
case uint32:
parsedValue = int64(v)
case string:
@ -86,5 +88,5 @@ func (c *intCondition) check() error {
}
func (c *intCondition) string() string {
return fmt.Sprintf("%s %s %d", c.key, getOpName(c.operator), c.value)
return fmt.Sprintf("%s %s %d", escapeString(c.key), getOpName(c.operator), c.value)
}

2
database/query/condition-regex.go
View file

@ -57,5 +57,5 @@ func (c *regexCondition) check() error {
}
func (c *regexCondition) string() string {
return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.regex.String())
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.regex.String()))
}

2
database/query/condition-string.go
View file

@ -56,5 +56,5 @@ func (c *stringCondition) check() error {
}
func (c *stringCondition) string() string {
return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.value)
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.value))
}

10
database/query/condition-stringslice.go
View file

@ -20,7 +20,7 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
parsedValue := strings.Split(v, ",")
if len(parsedValue) < 2 {
return &stringSliceCondition{
key: fmt.Sprintf("could not parse \"%s\" to []string", v),
key: v,
operator: errorPresent,
}
}
@ -29,6 +29,12 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
operator: operator,
value: parsedValue,
}
case []string:
return &stringSliceCondition{
key: key,
operator: operator,
value: v,
}
default:
return &stringSliceCondition{
key: fmt.Sprintf("incompatible value %v for []string", value),
@ -60,5 +66,5 @@ func (c *stringSliceCondition) check() error {
}
func (c *stringSliceCondition) string() string {
return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), strings.Join(c.value, ","))
return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(strings.Join(c.value, ",")))
}

76
database/query/condition_test.go Normal file
View file

@ -0,0 +1,76 @@
package query
import "testing"
func testSuccess(t *testing.T, c Condition) {
err := c.check()
if err != nil {
t.Errorf("failed: %s", err)
}
}
func TestInterfaces(t *testing.T) {
testSuccess(t, newIntCondition("banana", Equals, uint(1)))
testSuccess(t, newIntCondition("banana", Equals, uint8(1)))
testSuccess(t, newIntCondition("banana", Equals, uint16(1)))
testSuccess(t, newIntCondition("banana", Equals, uint32(1)))
testSuccess(t, newIntCondition("banana", Equals, int(1)))
testSuccess(t, newIntCondition("banana", Equals, int8(1)))
testSuccess(t, newIntCondition("banana", Equals, int16(1)))
testSuccess(t, newIntCondition("banana", Equals, int32(1)))
testSuccess(t, newIntCondition("banana", Equals, int64(1)))
testSuccess(t, newIntCondition("banana", Equals, "1"))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint8(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint16(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, uint32(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int8(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int16(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int32(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, int64(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, float32(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, float64(1)))
testSuccess(t, newFloatCondition("banana", FloatEquals, "1.1"))
testSuccess(t, newStringCondition("banana", SameAs, "coconut"))
testSuccess(t, newRegexCondition("banana", Matches, "coconut"))
testSuccess(t, newStringSliceCondition("banana", FloatEquals, []string{"banana", "coconut"}))
testSuccess(t, newStringSliceCondition("banana", FloatEquals, "banana,coconut"))
}
func testCondError(t *testing.T, c Condition) {
err := c.check()
if err == nil {
t.Error("should fail")
}
}
func TestConditionErrors(t *testing.T) {
// test invalid value types
testCondError(t, newBoolCondition("banana", Is, 1))
testCondError(t, newFloatCondition("banana", FloatEquals, true))
testCondError(t, newIntCondition("banana", Equals, true))
testCondError(t, newStringCondition("banana", SameAs, 1))
testCondError(t, newRegexCondition("banana", Matches, 1))
testCondError(t, newStringSliceCondition("banana", Matches, 1))
// test error presence
testCondError(t, newBoolCondition("banana", errorPresent, true))
testCondError(t, And(newBoolCondition("banana", errorPresent, true)))
testCondError(t, Or(newBoolCondition("banana", errorPresent, true)))
testCondError(t, newExistsCondition("banana", errorPresent))
testCondError(t, newFloatCondition("banana", errorPresent, 1.1))
testCondError(t, newIntCondition("banana", errorPresent, 1))
testCondError(t, newStringCondition("banana", errorPresent, "coconut"))
testCondError(t, newRegexCondition("banana", errorPresent, "coconut"))
}
func TestWhere(t *testing.T) {
c := Where("", 254, nil)
err := c.check()
if err == nil {
t.Error("should fail")
}
}

53
database/query/operators.go Normal file
View file

@ -0,0 +1,53 @@
package query
var (
operatorNames = map[string]uint8{
"==": Equals,
">": GreaterThan,
">=": GreaterThanOrEqual,
"<": LessThan,
"<=": LessThanOrEqual,
"f==": FloatEquals,
"f>": FloatGreaterThan,
"f>=": FloatGreaterThanOrEqual,
"f<": FloatLessThan,
"f<=": FloatLessThanOrEqual,
"sameas": SameAs,
"s==": SameAs,
"contains": Contains,
"co": Contains,
"startswith": StartsWith,
"sw": StartsWith,
"endswith": EndsWith,
"ew": EndsWith,
"in": In,
"matches": Matches,
"re": Matches,
"is": Is,
"exists": Exists,
"ex": Exists,
}
primaryNames = make(map[uint8]string)
)
func init() {
for opName, opID := range operatorNames {
name, ok := primaryNames[opID]
if ok {
if len(name) < len(opName) {
primaryNames[opID] = opName
}
} else {
primaryNames[opID] = opName
}
}
}
func getOpName(operator uint8) string {
name, ok := primaryNames[operator]
if ok {
return name
}
return "[unknown]"
}

9
database/query/operators_test.go Normal file
View file

@ -0,0 +1,9 @@
package query
import "testing"
func TestGetOpName(t *testing.T) {
if getOpName(254) != "[unknown]" {
t.Error("unexpected output")
}
}

388
database/query/parser.go
View file

@ -4,77 +4,122 @@ import (
"errors"
"fmt"
"regexp"
"strconv"
"strings"
)
var (
operatorNames = map[string]uint8{
"==": Equals,
">": GreaterThan,
">=": GreaterThanOrEqual,
"<": LessThan,
"<=": LessThanOrEqual,
"f==": FloatEquals,
"f>": FloatGreaterThan,
"f>=": FloatGreaterThanOrEqual,
"f<": FloatLessThan,
"f<=": FloatLessThanOrEqual,
"sameas": SameAs,
"s==": SameAs,
"contains": Contains,
"co": Contains,
"startswith": StartsWith,
"sw": StartsWith,
"endswith": EndsWith,
"ew": EndsWith,
"in": In,
"matches": Matches,
"re": Matches,
"is": Is,
"exists": Exists,
"ex": Exists,
}
primaryNames = make(map[uint8]string)
)
func init() {
for opName, opID := range operatorNames {
name, ok := primaryNames[opID]
if ok {
if len(name) < len(opName) {
primaryNames[opID] = opName
}
} else {
primaryNames[opID] = opName
}
}
}
func getOpName(operator uint8) string {
name, ok := primaryNames[operator]
if ok {
return name
}
return "[unknown]"
}
type treeElement struct {
branches []*treeElement
type snippet struct {
text string
globalPosition int
}
var (
escapeReplacer = regexp.MustCompile("\\\\([^\\\\])")
)
// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
func ParseQuery(query string) (*Query, error) {
snippets, err := extractSnippets(query)
if err != nil {
return nil, err
}
snippetsPos := 0
// prepToken removes surrounding parenthesis and escape characters.
func prepToken(text string) string {
return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
getSnippet := func() (*snippet, error) {
// order is important, as parseAndOr will always consume one additional snippet.
snippetsPos++
if snippetsPos > len(snippets) {
return nil, fmt.Errorf("unexpected end at position %d", len(query))
}
return snippets[snippetsPos-1], nil
}
remainingSnippets := func() int {
return len(snippets) - snippetsPos
}
// check for query word
queryWord, err := getSnippet()
if err != nil {
return nil, err
}
if queryWord.text != "query" {
return nil, errors.New("queries must start with \"query\"")
}
// get prefix
prefix, err := getSnippet()
if err != nil {
return nil, err
}
q := New(prefix.text)
for remainingSnippets() > 0 {
command, err := getSnippet()
if err != nil {
return nil, err
}
switch command.text {
case "where":
if q.where != nil {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
// parse conditions
condition, err := parseAndOr(getSnippet, remainingSnippets, true)
if err != nil {
return nil, err
}
// go one back, as parseAndOr had to check if its done
snippetsPos--
q.Where(condition)
case "orderby":
if q.orderBy != "" {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
orderBySnippet, err := getSnippet()
if err != nil {
return nil, err
}
q.OrderBy(orderBySnippet.text)
case "limit":
if q.limit != 0 {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
limitSnippet, err := getSnippet()
if err != nil {
return nil, err
}
limit, err := strconv.ParseUint(limitSnippet.text, 10, 31)
if err != nil {
return nil, fmt.Errorf("could not parse integer (%s) at position %d", limitSnippet.text, limitSnippet.globalPosition)
}
q.Limit(int(limit))
case "offset":
if q.offset != 0 {
return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
}
offsetSnippet, err := getSnippet()
if err != nil {
return nil, err
}
offset, err := strconv.ParseUint(offsetSnippet.text, 10, 31)
if err != nil {
return nil, fmt.Errorf("could not parse integer (%s) at position %d", offsetSnippet.text, offsetSnippet.globalPosition)
}
q.Offset(int(offset))
default:
return nil, fmt.Errorf("unknown clause \"%s\" at position %d", command.text, command.globalPosition)
}
}
return q.Check()
}
func extractSnippets(text string) (snippets []*treeElement, err error) {
func extractSnippets(text string) (snippets []*snippet, err error) {
skip := false
start := -1
@ -93,10 +138,10 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
skip = true
}
// wait for parenthesis to be over
// wait for parenthesis to be overs
if inParenthesis {
if char == '"' {
snippets = append(snippets, &treeElement{
snippets = append(snippets, &snippet{
text: prepToken(text[start+1 : pos]),
globalPosition: start + 1,
})
@ -110,9 +155,9 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
switch char {
case '\t', '\n', '\r', ' ', '(', ')':
if start >= 0 {
snippets = append(snippets, &treeElement{
snippets = append(snippets, &snippet{
text: prepToken(text[start:pos]),
globalPosition: start,
globalPosition: start + 1,
})
start = -1
}
@ -125,13 +170,13 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
// handle special segment characters
switch char {
case '(', ')':
snippets = append(snippets, &treeElement{
snippets = append(snippets, &snippet{
text: text[pos : pos+1],
globalPosition: pos,
globalPosition: pos + 1,
})
case '"':
if start < pos {
return nil, fmt.Errorf("parenthesis ('\"') may not be within words, please escape with '\\' (position: %d)", pos+1)
return nil, fmt.Errorf("parenthesis ('\"') may not be used within words, please escape with '\\' (position: %d)", pos+1)
}
inParenthesis = true
}
@ -139,89 +184,166 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
}
// add last
snippets = append(snippets, &treeElement{
text: prepToken(text[start : pos+1]),
globalPosition: start,
})
if start >= 0 {
snippets = append(snippets, &snippet{
text: prepToken(text[start : pos+1]),
globalPosition: start + 1,
})
}
return snippets, nil
}
// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
func ParseQuery(query string) (*Query, error) {
snippets, err := extractSnippets(query)
if err != nil {
return nil, err
}
snippetsPos := 0
func parseAndOr(getSnippet func() (*snippet, error), remainingSnippets func() int, rootCondition bool) (Condition, error) {
var isOr = false
var typeSet = false
var wrapInNot = false
var expectingMore = true
var conditions []Condition
getElement := func() (*treeElement, error) {
if snippetsPos >= len(snippets) {
return nil, fmt.Errorf("unexpected end at position %d", len(query))
for {
if !expectingMore && rootCondition && remainingSnippets() == 0 {
// advance snippetsPos by one, as it will be set back by 1
getSnippet()
if len(conditions) == 1 {
return conditions[0], nil
}
if isOr {
return Or(conditions...), nil
}
return And(conditions...), nil
}
return snippets[snippetsPos], nil
}
// check for query word
queryWord, err := getElement()
if err != nil {
return nil, err
}
if queryWord.text != "query" {
return nil, errors.New("queries must start with \"query\"")
}
firstSnippet, err := getSnippet()
if err != nil {
return nil, err
}
// get prefix
prefix, err := getElement()
if err != nil {
return nil, err
}
if !expectingMore && rootCondition {
switch firstSnippet.text {
case "orderby", "limit", "offset":
if len(conditions) == 1 {
return conditions[0], nil
}
if isOr {
return Or(conditions...), nil
}
return And(conditions...), nil
}
}
// check if no condition
if len(snippets) == 2 {
return New(prefix.text, nil)
switch firstSnippet.text {
case "(":
condition, err := parseAndOr(getSnippet, remainingSnippets, false)
if err != nil {
return nil, err
}
if wrapInNot {
conditions = append(conditions, Not(condition))
wrapInNot = false
} else {
conditions = append(conditions, condition)
}
expectingMore = true
case ")":
if len(conditions) == 1 {
return conditions[0], nil
}
if isOr {
return Or(conditions...), nil
}
return And(conditions...), nil
case "and":
if typeSet && isOr {
return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
}
isOr = false
typeSet = true
expectingMore = true
case "or":
if typeSet && !isOr {
return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
}
isOr = true
typeSet = true
expectingMore = true
case "not":
wrapInNot = true
expectingMore = true
default:
condition, err := parseCondition(firstSnippet, getSnippet)
if err != nil {
return nil, err
}
if wrapInNot {
conditions = append(conditions, Not(condition))
wrapInNot = false
} else {
conditions = append(conditions, condition)
}
expectingMore = false
}
}
// check for where word
whereWord, err := getElement()
if err != nil {
return nil, err
}
if whereWord.text != "where" {
return nil, errors.New("filtering queries must start conditions with \"where\"")
}
// parse conditions
condition, err := parseCondition(getElement)
if err != nil {
return nil, err
}
// check for additional tokens
// token := s.Scan()
// if token != scanner.EOF {
// return nil, fmt.Errorf("unexpected additional tokens at position %d", s.Position)
// }
return New(prefix.text, condition)
}
func parseCondition(getElement func() (*treeElement, error)) (Condition, error) {
first, err := getElement()
func parseCondition(firstSnippet *snippet, getSnippet func() (*snippet, error)) (Condition, error) {
wrapInNot := false
// get operator name
opName, err := getSnippet()
if err != nil {
return nil, err
}
switch first.text {
case "(":
return parseAndOr(getElement, true, nil, false)
// case ""
// negate?
if opName.text == "not" {
wrapInNot = true
opName, err = getSnippet()
if err != nil {
return nil, err
}
}
return nil, nil
// get operator
operator, ok := operatorNames[opName.text]
if !ok {
return nil, fmt.Errorf("unknown operator at position %d", opName.globalPosition)
}
// don't need a value for "exists"
if operator == Exists {
if wrapInNot {
return Not(Where(firstSnippet.text, operator, nil)), nil
}
return Where(firstSnippet.text, operator, nil), nil
}
// get value
value, err := getSnippet()
if err != nil {
return nil, err
}
if wrapInNot {
return Not(Where(firstSnippet.text, operator, value.text)), nil
}
return Where(firstSnippet.text, operator, value.text), nil
}
func parseAndOr(getElement func() (*treeElement, error), expectBracket bool, preParsedCondition Condition, preParsedIsOr bool) (Condition, error) {
return nil, nil
var (
escapeReplacer = regexp.MustCompile("\\\\([^\\\\])")
)
// prepToken removes surrounding parenthesis and escape characters.
func prepToken(text string) string {
return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
}
// escapeString correctly escapes a snippet for printing
func escapeString(token string) string {
// check if token contains characters that need to be escaped
if strings.ContainsAny(token, "()\"\\\t\r\n ") {
// put the token in parenthesis and only escape \ and "
return fmt.Sprintf("\"%s\"", strings.Replace(token, "\"", "\\\"", -1))
}
return token
}

158
database/query/parser_test.go
View file

@ -1,18 +1,168 @@
package query
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
)
func TestMakeTree(t *testing.T) {
text1 := `(age > 100 and experience <= "99") or (age < 10 and motivation > 50) or name matches Mr.\"X or name matches y`
func TestExtractSnippets(t *testing.T) {
text1 := `query test: where ( "bananas" > 100 and monkeys.# <= "12")or(coconuts < 10 "and" area > 50) or name sameas Julian or name matches ^King\ `
result1 := []*snippet{
&snippet{text: "query", globalPosition: 1},
&snippet{text: "test:", globalPosition: 7},
&snippet{text: "where", globalPosition: 13},
&snippet{text: "(", globalPosition: 19},
&snippet{text: "bananas", globalPosition: 21},
&snippet{text: ">", globalPosition: 31},
&snippet{text: "100", globalPosition: 33},
&snippet{text: "and", globalPosition: 37},
&snippet{text: "monkeys.#", globalPosition: 41},
&snippet{text: "<=", globalPosition: 51},
&snippet{text: "12", globalPosition: 54},
&snippet{text: ")", globalPosition: 58},
&snippet{text: "or", globalPosition: 59},
&snippet{text: "(", globalPosition: 61},
&snippet{text: "coconuts", globalPosition: 62},
&snippet{text: "<", globalPosition: 71},
&snippet{text: "10", globalPosition: 73},
&snippet{text: "and", globalPosition: 76},
&snippet{text: "area", globalPosition: 82},
&snippet{text: ">", globalPosition: 87},
&snippet{text: "50", globalPosition: 89},
&snippet{text: ")", globalPosition: 91},
&snippet{text: "or", globalPosition: 93},
&snippet{text: "name", globalPosition: 96},
&snippet{text: "sameas", globalPosition: 101},
&snippet{text: "Julian", globalPosition: 108},
&snippet{text: "or", globalPosition: 115},
&snippet{text: "name", globalPosition: 118},
&snippet{text: "matches", globalPosition: 123},
&snippet{text: "^King ", globalPosition: 131},
}
snippets, err := extractSnippets(text1)
if err != nil {
t.Errorf("failed to make tree: %s", err)
} else {
t.Errorf("failed to extract snippets: %s", err)
}
if !reflect.DeepEqual(result1, snippets) {
t.Errorf("unexpected results:")
for _, el := range snippets {
t.Errorf("%+v", el)
}
}
// t.Error(spew.Sprintf("%v", treeElement))
}
func testParsing(t *testing.T, queryText string, expectedResult *Query) {
_, err := expectedResult.Check()
if err != nil {
t.Errorf("failed to create query: %s", err)
return
}
q, err := ParseQuery(queryText)
if err != nil {
t.Errorf("failed to parse query: %s", err)
return
}
if queryText != q.Print() {
t.Errorf("string match failed: %s", q.Print())
return
}
if !reflect.DeepEqual(expectedResult, q) {
t.Error("deepqual match failed.")
t.Error("got:")
t.Error(spew.Sdump(q))
t.Error("expected:")
t.Error(spew.Sdump(expectedResult))
}
}
func TestParseQuery(t *testing.T) {
text1 := `query test: where (bananas > 100 and monkeys.# <= 12) or not (coconuts < 10 and area not > 50) or name sameas Julian or name matches "^King " orderby name limit 10 offset 20`
result1 := New("test:").Where(Or(
And(
Where("bananas", GreaterThan, 100),
Where("monkeys.#", LessThanOrEqual, 12),
),
Not(And(
Where("coconuts", LessThan, 10),
Not(Where("area", GreaterThan, 50)),
)),
Where("name", SameAs, "Julian"),
Where("name", Matches, "^King "),
)).OrderBy("name").Limit(10).Offset(20)
testParsing(t, text1, result1)
testParsing(t, `query test: orderby name`, New("test:").OrderBy("name"))
testParsing(t, `query test: limit 10`, New("test:").Limit(10))
testParsing(t, `query test: offset 10`, New("test:").Offset(10))
testParsing(t, `query test: where banana matches ^ban`, New("test:").Where(Where("banana", Matches, "^ban")))
testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
// test all operators
testParsing(t, `query test: where banana == 1`, New("test:").Where(Where("banana", Equals, 1)))
testParsing(t, `query test: where banana > 1`, New("test:").Where(Where("banana", GreaterThan, 1)))
testParsing(t, `query test: where banana >= 1`, New("test:").Where(Where("banana", GreaterThanOrEqual, 1)))
testParsing(t, `query test: where banana < 1`, New("test:").Where(Where("banana", LessThan, 1)))
testParsing(t, `query test: where banana <= 1`, New("test:").Where(Where("banana", LessThanOrEqual, 1)))
testParsing(t, `query test: where banana f== 1.1`, New("test:").Where(Where("banana", FloatEquals, 1.1)))
testParsing(t, `query test: where banana f> 1.1`, New("test:").Where(Where("banana", FloatGreaterThan, 1.1)))
testParsing(t, `query test: where banana f>= 1.1`, New("test:").Where(Where("banana", FloatGreaterThanOrEqual, 1.1)))
testParsing(t, `query test: where banana f< 1.1`, New("test:").Where(Where("banana", FloatLessThan, 1.1)))
testParsing(t, `query test: where banana f<= 1.1`, New("test:").Where(Where("banana", FloatLessThanOrEqual, 1.1)))
testParsing(t, `query test: where banana sameas banana`, New("test:").Where(Where("banana", SameAs, "banana")))
testParsing(t, `query test: where banana contains banana`, New("test:").Where(Where("banana", Contains, "banana")))
testParsing(t, `query test: where banana startswith banana`, New("test:").Where(Where("banana", StartsWith, "banana")))
testParsing(t, `query test: where banana endswith banana`, New("test:").Where(Where("banana", EndsWith, "banana")))
testParsing(t, `query test: where banana in banana,coconut`, New("test:").Where(Where("banana", In, []string{"banana", "coconut"})))
testParsing(t, `query test: where banana matches banana`, New("test:").Where(Where("banana", Matches, "banana")))
testParsing(t, `query test: where banana is true`, New("test:").Where(Where("banana", Is, true)))
testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
// special
testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
}
func testParseError(t *testing.T, queryText string, expectedErrorString string) {
_, err := ParseQuery(queryText)
if err == nil {
t.Errorf("should fail to parse: %s", queryText)
return
}
if err.Error() != expectedErrorString {
t.Errorf("unexpected error for query: %s\nwanted: %s\n got: %s", queryText, expectedErrorString, err)
}
}
func TestParseErrors(t *testing.T) {
// syntax
testParseError(t, `query`, `unexpected end at position 5`)
testParseError(t, `query test`, `invalid prefix: test`)
testParseError(t, `query test: where`, `unexpected end at position 17`)
testParseError(t, `query test: where (`, `unexpected end at position 19`)
testParseError(t, `query test: where )`, `unknown clause ")" at position 19`)
testParseError(t, `query test: where not`, `unexpected end at position 21`)
testParseError(t, `query test: where banana`, `unexpected end at position 24`)
testParseError(t, `query test: where banana >`, `unexpected end at position 26`)
testParseError(t, `query test: where banana nope`, `unknown operator at position 26`)
testParseError(t, `query test: where banana exists or`, `unexpected end at position 34`)
testParseError(t, `query test: where banana exists and`, `unexpected end at position 35`)
testParseError(t, `query test: where banana exists and (`, `unexpected end at position 37`)
testParseError(t, `query test: where banana exists and banana is true or`, `you may not mix "and" and "or" (position: 52)`)
testParseError(t, `query test: where banana exists or banana is true and`, `you may not mix "and" and "or" (position: 51)`)
// testParseError(t, `query test: where banana exists and (`, ``)
// value parsing error
testParseError(t, `query test: where banana == banana`, `could not parse banana to int64: strconv.ParseInt: parsing "banana": invalid syntax (hint: use "sameas" to compare strings)`)
testParseError(t, `query test: where banana f== banana`, `could not parse banana to float64: strconv.ParseFloat: parsing "banana": invalid syntax`)
testParseError(t, `query test: where banana in banana`, `could not parse "banana" to []string`)
testParseError(t, `query test: where banana matches [banana`, "could not compile regex \"[banana\": error parsing regexp: missing closing ]: `[banana`")
testParseError(t, `query test: where banana is great`, `could not parse "great" to bool: strconv.ParseBool: parsing "great": invalid syntax`)
}

117
database/query/query.go
View file

@ -22,56 +22,113 @@ var (
// Query contains a compiled query.
type Query struct {
prefix string
condition Condition
checked bool
prefix string
where Condition
orderBy string
limit int
offset int
}
// New creates a new query.
func New(prefix string, condition Condition) (*Query, error) {
// New creates a new query with the supplied prefix.
func New(prefix string) *Query {
return &Query{
prefix: prefix,
}
}
// Where adds filtering.
func (q *Query) Where(condition Condition) *Query {
q.where = condition
return q
}
// Limit limits the number of returned results.
func (q *Query) Limit(limit int) *Query {
q.limit = limit
return q
}
// Offset sets the query offset.
func (q *Query) Offset(offset int) *Query {
q.offset = offset
return q
}
// OrderBy orders the results by the given key.
func (q *Query) OrderBy(key string) *Query {
q.orderBy = key
return q
}
// Check checks for errors in the query.
func (q *Query) Check() (*Query, error) {
if q.checked {
return q, nil
}
// check prefix
if !prefixExpr.MatchString(prefix) {
return nil, fmt.Errorf("invalid prefix: %s", prefix)
if !prefixExpr.MatchString(q.prefix) {
return nil, fmt.Errorf("invalid prefix: %s", q.prefix)
}
// check condition
if condition != nil {
err := condition.check()
if q.where != nil {
err := q.where.check()
if err != nil {
return nil, err
}
} else {
condition = &noCond{}
q.where = &noCond{}
}
// return query
return &Query{
prefix: prefix,
condition: condition,
}, nil
q.checked = true
return q, nil
}
// MustCompile creates a new query and panics on an error.
func MustCompile(prefix string, condition Condition) *Query {
q, err := New(prefix, condition)
// MustBeValid checks for errors in the query and panics if there is an error.
func (q *Query) MustBeValid() *Query {
_, err := q.Check()
if err != nil {
panic(err)
}
return q
}
// Matches checks whether the query matches the supplied data object.
func (q *Query) Matches(f Fetcher) bool {
return q.condition.complies(f)
// IsChecked returns whether they query was checked.
func (q *Query) IsChecked() bool {
return q.checked
}
// String returns the string representation of the query.
func (q *Query) String() string {
text := q.condition.string()
if text == "" {
return fmt.Sprintf("query %s", q.prefix)
}
if strings.HasPrefix(text, "(") {
text = text[1 : len(text)-1]
}
return fmt.Sprintf("query %s where %s", q.prefix, text)
// Matches checks whether the query matches the supplied data object.
func (q *Query) Matches(f Fetcher) bool {
return q.where.complies(f)
}
// Print returns the string representation of the query.
func (q *Query) Print() string {
where := q.where.string()
if where != "" {
if strings.HasPrefix(where, "(") {
where = where[1 : len(where)-1]
}
where = fmt.Sprintf(" where %s", where)
}
var orderBy string
if q.orderBy != "" {
orderBy = fmt.Sprintf(" orderby %s", q.orderBy)
}
var limit string
if q.limit > 0 {
limit = fmt.Sprintf(" limit %d", q.limit)
}
var offset string
if q.offset > 0 {
offset = fmt.Sprintf(" offset %d", q.offset)
}
return fmt.Sprintf("query %s%s%s%s%s", q.prefix, where, orderBy, limit, offset)
}

13
database/query/query_test.go
View file

@ -6,7 +6,7 @@ import (
var (
// copied from https://github.com/tidwall/gjson/blob/master/gjson_test.go
testJson = `{"age":100, "name":{"here":"B\\\"R"},
testJSON = `{"age":100, "name":{"here":"B\\\"R"},
"noop":{"what is a wren?":"a bird"},
"happy":true,"immortal":false,
"items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7],
@ -45,25 +45,25 @@ var (
)
func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) {
q := MustCompile("test:", condition)
q := New("test:").Where(condition).MustBeValid()
// fmt.Printf("%s\n", q.String())
matched := q.Matches(f)
switch {
case !matched && shouldMatch:
t.Errorf("should match: %s", q.String())
t.Errorf("should match: %s", q.Print())
case matched && !shouldMatch:
t.Errorf("should not match: %s", q.String())
t.Errorf("should not match: %s", q.Print())
}
}
func TestQuery(t *testing.T) {
// if !gjson.Valid(testJson) {
// if !gjson.Valid(testJSON) {
// t.Fatal("test json is invalid")
// }
f := NewJSONFetcher(testJson)
f := NewJSONFetcher(testJSON)
testQuery(t, f, true, Where("age", Equals, 100))
testQuery(t, f, true, Where("age", GreaterThan, uint8(99)))
@ -88,6 +88,7 @@ func TestQuery(t *testing.T) {
testQuery(t, f, true, Where("lastly.yay", In, "draft,final"))
testQuery(t, f, true, Where("lastly.yay", In, "final,draft"))
testQuery(t, f, true, Where("happy", Is, true))
testQuery(t, f, true, Where("happy", Is, "true"))
testQuery(t, f, true, Where("happy", Is, "t"))
testQuery(t, f, true, Not(Where("happy", Is, "0")))