diff --git a/database/query/README.md b/database/query/README.md new file mode 100644 index 0000000..9311417 --- /dev/null +++ b/database/query/README.md @@ -0,0 +1,55 @@ +# Query + +## Control Flow + +- Grouping with `(` and `)` +- Chaining with `and` and `or` + - _NO_ mixing! Be explicit and use grouping. +- Negation with `not` + - in front of expression for group: `not (...)` + - inside expression for clause: `name not matches "^King "` + +## Selectors + +Supported by all feeders: +- root level field: `field` +- sub level field: `field.sub` +- array/slice/map access: `map.0` +- array/slice/map length: `map.#` + +Please note that some feeders may have other special characters. It is advised to only use alphanumeric characters for keys. + +## Operators + +| Name | Textual | Req. Type | Internal Type | Compared with | +|---|---|---|---| +| Equals | `==` | int | int64 | `==` | +| GreaterThan | `>` | int | int64 | `>` | +| GreaterThanOrEqual | `>=` | int | int64 | `>=` | +| LessThan | `<` | int | int64 | `<` | +| LessThanOrEqual | `<=` | int | int64 | `<=` | +| FloatEquals | `f==` | float | float64 | `==` | +| FloatGreaterThan | `f>` | float | float64 | `>` | +| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` | +| FloatLessThan | `f<` | float | float64 | `<` | +| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` | +| SameAs | `sameas`, `s==` | string | string | `==` | +| Contains | `contains`, `co` | string | string | `strings.Contains()` | +| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` | +| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` | +| In | `in` | string | string | for loop with `==` | +| Matches | `matches`, `re` | string | int64 | `regexp.Regexp.Matches()` | +| Is | `is` | bool* | bool | `==` | +| Exists | `exists`, `ex` | any | n/a | n/a | + +\*accepts strings: 1, t, T, TRUE, true, True, 0, f, F, FALSE + +## Escaping + +If you need to use a control character within a value (ie. not for controlling), escape it with `\`. +It is recommended to wrap a word into parenthesis instead of escaping control characters, when possible. + +| Location | Characters to be escaped | +|---|---| +| Within parenthesis (`"`) | `"`, `\` | +| Everywhere else | `(`, `)`, `"`, `\`, `\t`, `\r`, `\n`, ` ` (space) | diff --git a/database/query/condition-bool.go b/database/query/condition-bool.go index 49f1608..e9d7c2b 100644 --- a/database/query/condition-bool.go +++ b/database/query/condition-bool.go @@ -64,5 +64,5 @@ func (c *boolCondition) check() error { } func (c *boolCondition) string() string { - return fmt.Sprintf("%s %s %t", c.key, getOpName(c.operator), c.value) + return fmt.Sprintf("%s %s %t", escapeString(c.key), getOpName(c.operator), c.value) } diff --git a/database/query/condition-exists.go b/database/query/condition-exists.go index bde67a8..2e2b013 100644 --- a/database/query/condition-exists.go +++ b/database/query/condition-exists.go @@ -29,5 +29,5 @@ func (c *existsCondition) check() error { } func (c *existsCondition) string() string { - return fmt.Sprintf("%s %s", c.key, getOpName(c.operator)) + return fmt.Sprintf("%s %s", escapeString(c.key), getOpName(c.operator)) } diff --git a/database/query/condition-float.go b/database/query/condition-float.go index 851b3f2..c34cf7f 100644 --- a/database/query/condition-float.go +++ b/database/query/condition-float.go @@ -31,6 +31,8 @@ func newFloatCondition(key string, operator uint8, value interface{}) *floatCond parsedValue = float64(v) case uint8: parsedValue = float64(v) + case uint16: + parsedValue = float64(v) case uint32: parsedValue = float64(v) case float32: @@ -90,5 +92,5 @@ func (c *floatCondition) check() error { } func (c *floatCondition) string() string { - return fmt.Sprintf("%s %s %f", c.key, getOpName(c.operator), c.value) + return fmt.Sprintf("%s %s %g", escapeString(c.key), getOpName(c.operator), c.value) } diff --git a/database/query/condition-int.go b/database/query/condition-int.go index eaf427b..8f18ab2 100644 --- a/database/query/condition-int.go +++ b/database/query/condition-int.go @@ -31,6 +31,8 @@ func newIntCondition(key string, operator uint8, value interface{}) *intConditio parsedValue = int64(v) case uint8: parsedValue = int64(v) + case uint16: + parsedValue = int64(v) case uint32: parsedValue = int64(v) case string: @@ -86,5 +88,5 @@ func (c *intCondition) check() error { } func (c *intCondition) string() string { - return fmt.Sprintf("%s %s %d", c.key, getOpName(c.operator), c.value) + return fmt.Sprintf("%s %s %d", escapeString(c.key), getOpName(c.operator), c.value) } diff --git a/database/query/condition-regex.go b/database/query/condition-regex.go index b6c2e8b..d795c84 100644 --- a/database/query/condition-regex.go +++ b/database/query/condition-regex.go @@ -57,5 +57,5 @@ func (c *regexCondition) check() error { } func (c *regexCondition) string() string { - return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.regex.String()) + return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.regex.String())) } diff --git a/database/query/condition-string.go b/database/query/condition-string.go index 8890b3c..747e337 100644 --- a/database/query/condition-string.go +++ b/database/query/condition-string.go @@ -56,5 +56,5 @@ func (c *stringCondition) check() error { } func (c *stringCondition) string() string { - return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.value) + return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.value)) } diff --git a/database/query/condition-stringslice.go b/database/query/condition-stringslice.go index f8597cc..ab3004d 100644 --- a/database/query/condition-stringslice.go +++ b/database/query/condition-stringslice.go @@ -20,7 +20,7 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str parsedValue := strings.Split(v, ",") if len(parsedValue) < 2 { return &stringSliceCondition{ - key: fmt.Sprintf("could not parse \"%s\" to []string", v), + key: v, operator: errorPresent, } } @@ -29,6 +29,12 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str operator: operator, value: parsedValue, } + case []string: + return &stringSliceCondition{ + key: key, + operator: operator, + value: v, + } default: return &stringSliceCondition{ key: fmt.Sprintf("incompatible value %v for []string", value), @@ -60,5 +66,5 @@ func (c *stringSliceCondition) check() error { } func (c *stringSliceCondition) string() string { - return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), strings.Join(c.value, ",")) + return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(strings.Join(c.value, ","))) } diff --git a/database/query/condition_test.go b/database/query/condition_test.go new file mode 100644 index 0000000..eb871a7 --- /dev/null +++ b/database/query/condition_test.go @@ -0,0 +1,76 @@ +package query + +import "testing" + +func testSuccess(t *testing.T, c Condition) { + err := c.check() + if err != nil { + t.Errorf("failed: %s", err) + } +} + +func TestInterfaces(t *testing.T) { + testSuccess(t, newIntCondition("banana", Equals, uint(1))) + testSuccess(t, newIntCondition("banana", Equals, uint8(1))) + testSuccess(t, newIntCondition("banana", Equals, uint16(1))) + testSuccess(t, newIntCondition("banana", Equals, uint32(1))) + testSuccess(t, newIntCondition("banana", Equals, int(1))) + testSuccess(t, newIntCondition("banana", Equals, int8(1))) + testSuccess(t, newIntCondition("banana", Equals, int16(1))) + testSuccess(t, newIntCondition("banana", Equals, int32(1))) + testSuccess(t, newIntCondition("banana", Equals, int64(1))) + testSuccess(t, newIntCondition("banana", Equals, "1")) + + testSuccess(t, newFloatCondition("banana", FloatEquals, uint(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, uint8(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, uint16(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, uint32(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, int(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, int8(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, int16(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, int32(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, int64(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, float32(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, float64(1))) + testSuccess(t, newFloatCondition("banana", FloatEquals, "1.1")) + + testSuccess(t, newStringCondition("banana", SameAs, "coconut")) + testSuccess(t, newRegexCondition("banana", Matches, "coconut")) + testSuccess(t, newStringSliceCondition("banana", FloatEquals, []string{"banana", "coconut"})) + testSuccess(t, newStringSliceCondition("banana", FloatEquals, "banana,coconut")) +} + +func testCondError(t *testing.T, c Condition) { + err := c.check() + if err == nil { + t.Error("should fail") + } +} + +func TestConditionErrors(t *testing.T) { + // test invalid value types + testCondError(t, newBoolCondition("banana", Is, 1)) + testCondError(t, newFloatCondition("banana", FloatEquals, true)) + testCondError(t, newIntCondition("banana", Equals, true)) + testCondError(t, newStringCondition("banana", SameAs, 1)) + testCondError(t, newRegexCondition("banana", Matches, 1)) + testCondError(t, newStringSliceCondition("banana", Matches, 1)) + + // test error presence + testCondError(t, newBoolCondition("banana", errorPresent, true)) + testCondError(t, And(newBoolCondition("banana", errorPresent, true))) + testCondError(t, Or(newBoolCondition("banana", errorPresent, true))) + testCondError(t, newExistsCondition("banana", errorPresent)) + testCondError(t, newFloatCondition("banana", errorPresent, 1.1)) + testCondError(t, newIntCondition("banana", errorPresent, 1)) + testCondError(t, newStringCondition("banana", errorPresent, "coconut")) + testCondError(t, newRegexCondition("banana", errorPresent, "coconut")) +} + +func TestWhere(t *testing.T) { + c := Where("", 254, nil) + err := c.check() + if err == nil { + t.Error("should fail") + } +} diff --git a/database/query/operators.go b/database/query/operators.go new file mode 100644 index 0000000..bbd21ee --- /dev/null +++ b/database/query/operators.go @@ -0,0 +1,53 @@ +package query + +var ( + operatorNames = map[string]uint8{ + "==": Equals, + ">": GreaterThan, + ">=": GreaterThanOrEqual, + "<": LessThan, + "<=": LessThanOrEqual, + "f==": FloatEquals, + "f>": FloatGreaterThan, + "f>=": FloatGreaterThanOrEqual, + "f<": FloatLessThan, + "f<=": FloatLessThanOrEqual, + "sameas": SameAs, + "s==": SameAs, + "contains": Contains, + "co": Contains, + "startswith": StartsWith, + "sw": StartsWith, + "endswith": EndsWith, + "ew": EndsWith, + "in": In, + "matches": Matches, + "re": Matches, + "is": Is, + "exists": Exists, + "ex": Exists, + } + + primaryNames = make(map[uint8]string) +) + +func init() { + for opName, opID := range operatorNames { + name, ok := primaryNames[opID] + if ok { + if len(name) < len(opName) { + primaryNames[opID] = opName + } + } else { + primaryNames[opID] = opName + } + } +} + +func getOpName(operator uint8) string { + name, ok := primaryNames[operator] + if ok { + return name + } + return "[unknown]" +} diff --git a/database/query/operators_test.go b/database/query/operators_test.go new file mode 100644 index 0000000..3f4fe81 --- /dev/null +++ b/database/query/operators_test.go @@ -0,0 +1,9 @@ +package query + +import "testing" + +func TestGetOpName(t *testing.T) { + if getOpName(254) != "[unknown]" { + t.Error("unexpected output") + } +} diff --git a/database/query/parser.go b/database/query/parser.go index 2ceebd9..d14b847 100644 --- a/database/query/parser.go +++ b/database/query/parser.go @@ -4,77 +4,122 @@ import ( "errors" "fmt" "regexp" + "strconv" "strings" ) -var ( - operatorNames = map[string]uint8{ - "==": Equals, - ">": GreaterThan, - ">=": GreaterThanOrEqual, - "<": LessThan, - "<=": LessThanOrEqual, - "f==": FloatEquals, - "f>": FloatGreaterThan, - "f>=": FloatGreaterThanOrEqual, - "f<": FloatLessThan, - "f<=": FloatLessThanOrEqual, - "sameas": SameAs, - "s==": SameAs, - "contains": Contains, - "co": Contains, - "startswith": StartsWith, - "sw": StartsWith, - "endswith": EndsWith, - "ew": EndsWith, - "in": In, - "matches": Matches, - "re": Matches, - "is": Is, - "exists": Exists, - "ex": Exists, - } - - primaryNames = make(map[uint8]string) -) - -func init() { - for opName, opID := range operatorNames { - name, ok := primaryNames[opID] - if ok { - if len(name) < len(opName) { - primaryNames[opID] = opName - } - } else { - primaryNames[opID] = opName - } - } -} - -func getOpName(operator uint8) string { - name, ok := primaryNames[operator] - if ok { - return name - } - return "[unknown]" -} - -type treeElement struct { - branches []*treeElement +type snippet struct { text string globalPosition int } -var ( - escapeReplacer = regexp.MustCompile("\\\\([^\\\\])") -) +// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces. +func ParseQuery(query string) (*Query, error) { + snippets, err := extractSnippets(query) + if err != nil { + return nil, err + } + snippetsPos := 0 -// prepToken removes surrounding parenthesis and escape characters. -func prepToken(text string) string { - return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1") + getSnippet := func() (*snippet, error) { + // order is important, as parseAndOr will always consume one additional snippet. + snippetsPos++ + if snippetsPos > len(snippets) { + return nil, fmt.Errorf("unexpected end at position %d", len(query)) + } + return snippets[snippetsPos-1], nil + } + remainingSnippets := func() int { + return len(snippets) - snippetsPos + } + + // check for query word + queryWord, err := getSnippet() + if err != nil { + return nil, err + } + if queryWord.text != "query" { + return nil, errors.New("queries must start with \"query\"") + } + + // get prefix + prefix, err := getSnippet() + if err != nil { + return nil, err + } + q := New(prefix.text) + + for remainingSnippets() > 0 { + command, err := getSnippet() + if err != nil { + return nil, err + } + + switch command.text { + case "where": + if q.where != nil { + return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition) + } + + // parse conditions + condition, err := parseAndOr(getSnippet, remainingSnippets, true) + if err != nil { + return nil, err + } + // go one back, as parseAndOr had to check if its done + snippetsPos-- + + q.Where(condition) + case "orderby": + if q.orderBy != "" { + return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition) + } + + orderBySnippet, err := getSnippet() + if err != nil { + return nil, err + } + + q.OrderBy(orderBySnippet.text) + case "limit": + if q.limit != 0 { + return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition) + } + + limitSnippet, err := getSnippet() + if err != nil { + return nil, err + } + limit, err := strconv.ParseUint(limitSnippet.text, 10, 31) + if err != nil { + return nil, fmt.Errorf("could not parse integer (%s) at position %d", limitSnippet.text, limitSnippet.globalPosition) + } + + q.Limit(int(limit)) + case "offset": + if q.offset != 0 { + return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition) + } + + offsetSnippet, err := getSnippet() + if err != nil { + return nil, err + } + offset, err := strconv.ParseUint(offsetSnippet.text, 10, 31) + if err != nil { + return nil, fmt.Errorf("could not parse integer (%s) at position %d", offsetSnippet.text, offsetSnippet.globalPosition) + } + + q.Offset(int(offset)) + default: + return nil, fmt.Errorf("unknown clause \"%s\" at position %d", command.text, command.globalPosition) + } + } + + return q.Check() } -func extractSnippets(text string) (snippets []*treeElement, err error) { +func extractSnippets(text string) (snippets []*snippet, err error) { skip := false start := -1 @@ -93,10 +138,10 @@ func extractSnippets(text string) (snippets []*treeElement, err error) { skip = true } - // wait for parenthesis to be over + // wait for parenthesis to be overs if inParenthesis { if char == '"' { - snippets = append(snippets, &treeElement{ + snippets = append(snippets, &snippet{ text: prepToken(text[start+1 : pos]), globalPosition: start + 1, }) @@ -110,9 +155,9 @@ func extractSnippets(text string) (snippets []*treeElement, err error) { switch char { case '\t', '\n', '\r', ' ', '(', ')': if start >= 0 { - snippets = append(snippets, &treeElement{ + snippets = append(snippets, &snippet{ text: prepToken(text[start:pos]), - globalPosition: start, + globalPosition: start + 1, }) start = -1 } @@ -125,13 +170,13 @@ func extractSnippets(text string) (snippets []*treeElement, err error) { // handle special segment characters switch char { case '(', ')': - snippets = append(snippets, &treeElement{ + snippets = append(snippets, &snippet{ text: text[pos : pos+1], - globalPosition: pos, + globalPosition: pos + 1, }) case '"': if start < pos { - return nil, fmt.Errorf("parenthesis ('\"') may not be within words, please escape with '\\' (position: %d)", pos+1) + return nil, fmt.Errorf("parenthesis ('\"') may not be used within words, please escape with '\\' (position: %d)", pos+1) } inParenthesis = true } @@ -139,89 +184,166 @@ func extractSnippets(text string) (snippets []*treeElement, err error) { } // add last - snippets = append(snippets, &treeElement{ - text: prepToken(text[start : pos+1]), - globalPosition: start, - }) + if start >= 0 { + snippets = append(snippets, &snippet{ + text: prepToken(text[start : pos+1]), + globalPosition: start + 1, + }) + } return snippets, nil } -// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces. -func ParseQuery(query string) (*Query, error) { - snippets, err := extractSnippets(query) - if err != nil { - return nil, err - } - snippetsPos := 0 +func parseAndOr(getSnippet func() (*snippet, error), remainingSnippets func() int, rootCondition bool) (Condition, error) { + var isOr = false + var typeSet = false + var wrapInNot = false + var expectingMore = true + var conditions []Condition - getElement := func() (*treeElement, error) { - if snippetsPos >= len(snippets) { - return nil, fmt.Errorf("unexpected end at position %d", len(query)) + for { + if !expectingMore && rootCondition && remainingSnippets() == 0 { + // advance snippetsPos by one, as it will be set back by 1 + getSnippet() + if len(conditions) == 1 { + return conditions[0], nil + } + if isOr { + return Or(conditions...), nil + } + return And(conditions...), nil } - return snippets[snippetsPos], nil - } - // check for query word - queryWord, err := getElement() - if err != nil { - return nil, err - } - if queryWord.text != "query" { - return nil, errors.New("queries must start with \"query\"") - } + firstSnippet, err := getSnippet() + if err != nil { + return nil, err + } - // get prefix - prefix, err := getElement() - if err != nil { - return nil, err - } + if !expectingMore && rootCondition { + switch firstSnippet.text { + case "orderby", "limit", "offset": + if len(conditions) == 1 { + return conditions[0], nil + } + if isOr { + return Or(conditions...), nil + } + return And(conditions...), nil + } + } - // check if no condition - if len(snippets) == 2 { - return New(prefix.text, nil) + switch firstSnippet.text { + case "(": + condition, err := parseAndOr(getSnippet, remainingSnippets, false) + if err != nil { + return nil, err + } + if wrapInNot { + conditions = append(conditions, Not(condition)) + wrapInNot = false + } else { + conditions = append(conditions, condition) + } + expectingMore = true + case ")": + if len(conditions) == 1 { + return conditions[0], nil + } + if isOr { + return Or(conditions...), nil + } + return And(conditions...), nil + case "and": + if typeSet && isOr { + return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition) + } + isOr = false + typeSet = true + expectingMore = true + case "or": + if typeSet && !isOr { + return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition) + } + isOr = true + typeSet = true + expectingMore = true + case "not": + wrapInNot = true + expectingMore = true + default: + condition, err := parseCondition(firstSnippet, getSnippet) + if err != nil { + return nil, err + } + if wrapInNot { + conditions = append(conditions, Not(condition)) + wrapInNot = false + } else { + conditions = append(conditions, condition) + } + expectingMore = false + } } - - // check for where word - whereWord, err := getElement() - if err != nil { - return nil, err - } - if whereWord.text != "where" { - return nil, errors.New("filtering queries must start conditions with \"where\"") - } - - // parse conditions - condition, err := parseCondition(getElement) - if err != nil { - return nil, err - } - - // check for additional tokens - // token := s.Scan() - // if token != scanner.EOF { - // return nil, fmt.Errorf("unexpected additional tokens at position %d", s.Position) - // } - - return New(prefix.text, condition) } -func parseCondition(getElement func() (*treeElement, error)) (Condition, error) { - first, err := getElement() +func parseCondition(firstSnippet *snippet, getSnippet func() (*snippet, error)) (Condition, error) { + wrapInNot := false + + // get operator name + opName, err := getSnippet() if err != nil { return nil, err } - - switch first.text { - case "(": - return parseAndOr(getElement, true, nil, false) - // case "" + // negate? + if opName.text == "not" { + wrapInNot = true + opName, err = getSnippet() + if err != nil { + return nil, err + } } - return nil, nil + // get operator + operator, ok := operatorNames[opName.text] + if !ok { + return nil, fmt.Errorf("unknown operator at position %d", opName.globalPosition) + } + + // don't need a value for "exists" + if operator == Exists { + if wrapInNot { + return Not(Where(firstSnippet.text, operator, nil)), nil + } + return Where(firstSnippet.text, operator, nil), nil + } + + // get value + value, err := getSnippet() + if err != nil { + return nil, err + } + if wrapInNot { + return Not(Where(firstSnippet.text, operator, value.text)), nil + } + return Where(firstSnippet.text, operator, value.text), nil } -func parseAndOr(getElement func() (*treeElement, error), expectBracket bool, preParsedCondition Condition, preParsedIsOr bool) (Condition, error) { - return nil, nil +var ( + escapeReplacer = regexp.MustCompile("\\\\([^\\\\])") +) + +// prepToken removes surrounding parenthesis and escape characters. +func prepToken(text string) string { + return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1") +} + +// escapeString correctly escapes a snippet for printing +func escapeString(token string) string { + // check if token contains characters that need to be escaped + if strings.ContainsAny(token, "()\"\\\t\r\n ") { + // put the token in parenthesis and only escape \ and " + return fmt.Sprintf("\"%s\"", strings.Replace(token, "\"", "\\\"", -1)) + } + return token } diff --git a/database/query/parser_test.go b/database/query/parser_test.go index 53cbd31..e467100 100644 --- a/database/query/parser_test.go +++ b/database/query/parser_test.go @@ -1,18 +1,168 @@ package query import ( + "reflect" "testing" + + "github.com/davecgh/go-spew/spew" ) -func TestMakeTree(t *testing.T) { - text1 := `(age > 100 and experience <= "99") or (age < 10 and motivation > 50) or name matches Mr.\"X or name matches y` +func TestExtractSnippets(t *testing.T) { + text1 := `query test: where ( "bananas" > 100 and monkeys.# <= "12")or(coconuts < 10 "and" area > 50) or name sameas Julian or name matches ^King\ ` + result1 := []*snippet{ + &snippet{text: "query", globalPosition: 1}, + &snippet{text: "test:", globalPosition: 7}, + &snippet{text: "where", globalPosition: 13}, + &snippet{text: "(", globalPosition: 19}, + &snippet{text: "bananas", globalPosition: 21}, + &snippet{text: ">", globalPosition: 31}, + &snippet{text: "100", globalPosition: 33}, + &snippet{text: "and", globalPosition: 37}, + &snippet{text: "monkeys.#", globalPosition: 41}, + &snippet{text: "<=", globalPosition: 51}, + &snippet{text: "12", globalPosition: 54}, + &snippet{text: ")", globalPosition: 58}, + &snippet{text: "or", globalPosition: 59}, + &snippet{text: "(", globalPosition: 61}, + &snippet{text: "coconuts", globalPosition: 62}, + &snippet{text: "<", globalPosition: 71}, + &snippet{text: "10", globalPosition: 73}, + &snippet{text: "and", globalPosition: 76}, + &snippet{text: "area", globalPosition: 82}, + &snippet{text: ">", globalPosition: 87}, + &snippet{text: "50", globalPosition: 89}, + &snippet{text: ")", globalPosition: 91}, + &snippet{text: "or", globalPosition: 93}, + &snippet{text: "name", globalPosition: 96}, + &snippet{text: "sameas", globalPosition: 101}, + &snippet{text: "Julian", globalPosition: 108}, + &snippet{text: "or", globalPosition: 115}, + &snippet{text: "name", globalPosition: 118}, + &snippet{text: "matches", globalPosition: 123}, + &snippet{text: "^King ", globalPosition: 131}, + } + snippets, err := extractSnippets(text1) if err != nil { - t.Errorf("failed to make tree: %s", err) - } else { + t.Errorf("failed to extract snippets: %s", err) + } + + if !reflect.DeepEqual(result1, snippets) { + t.Errorf("unexpected results:") for _, el := range snippets { t.Errorf("%+v", el) } } + // t.Error(spew.Sprintf("%v", treeElement)) } + +func testParsing(t *testing.T, queryText string, expectedResult *Query) { + _, err := expectedResult.Check() + if err != nil { + t.Errorf("failed to create query: %s", err) + return + } + + q, err := ParseQuery(queryText) + if err != nil { + t.Errorf("failed to parse query: %s", err) + return + } + + if queryText != q.Print() { + t.Errorf("string match failed: %s", q.Print()) + return + } + if !reflect.DeepEqual(expectedResult, q) { + t.Error("deepqual match failed.") + t.Error("got:") + t.Error(spew.Sdump(q)) + t.Error("expected:") + t.Error(spew.Sdump(expectedResult)) + } +} + +func TestParseQuery(t *testing.T) { + text1 := `query test: where (bananas > 100 and monkeys.# <= 12) or not (coconuts < 10 and area not > 50) or name sameas Julian or name matches "^King " orderby name limit 10 offset 20` + result1 := New("test:").Where(Or( + And( + Where("bananas", GreaterThan, 100), + Where("monkeys.#", LessThanOrEqual, 12), + ), + Not(And( + Where("coconuts", LessThan, 10), + Not(Where("area", GreaterThan, 50)), + )), + Where("name", SameAs, "Julian"), + Where("name", Matches, "^King "), + )).OrderBy("name").Limit(10).Offset(20) + testParsing(t, text1, result1) + + testParsing(t, `query test: orderby name`, New("test:").OrderBy("name")) + testParsing(t, `query test: limit 10`, New("test:").Limit(10)) + testParsing(t, `query test: offset 10`, New("test:").Offset(10)) + testParsing(t, `query test: where banana matches ^ban`, New("test:").Where(Where("banana", Matches, "^ban"))) + testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil))) + testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil)))) + + // test all operators + testParsing(t, `query test: where banana == 1`, New("test:").Where(Where("banana", Equals, 1))) + testParsing(t, `query test: where banana > 1`, New("test:").Where(Where("banana", GreaterThan, 1))) + testParsing(t, `query test: where banana >= 1`, New("test:").Where(Where("banana", GreaterThanOrEqual, 1))) + testParsing(t, `query test: where banana < 1`, New("test:").Where(Where("banana", LessThan, 1))) + testParsing(t, `query test: where banana <= 1`, New("test:").Where(Where("banana", LessThanOrEqual, 1))) + testParsing(t, `query test: where banana f== 1.1`, New("test:").Where(Where("banana", FloatEquals, 1.1))) + testParsing(t, `query test: where banana f> 1.1`, New("test:").Where(Where("banana", FloatGreaterThan, 1.1))) + testParsing(t, `query test: where banana f>= 1.1`, New("test:").Where(Where("banana", FloatGreaterThanOrEqual, 1.1))) + testParsing(t, `query test: where banana f< 1.1`, New("test:").Where(Where("banana", FloatLessThan, 1.1))) + testParsing(t, `query test: where banana f<= 1.1`, New("test:").Where(Where("banana", FloatLessThanOrEqual, 1.1))) + testParsing(t, `query test: where banana sameas banana`, New("test:").Where(Where("banana", SameAs, "banana"))) + testParsing(t, `query test: where banana contains banana`, New("test:").Where(Where("banana", Contains, "banana"))) + testParsing(t, `query test: where banana startswith banana`, New("test:").Where(Where("banana", StartsWith, "banana"))) + testParsing(t, `query test: where banana endswith banana`, New("test:").Where(Where("banana", EndsWith, "banana"))) + testParsing(t, `query test: where banana in banana,coconut`, New("test:").Where(Where("banana", In, []string{"banana", "coconut"}))) + testParsing(t, `query test: where banana matches banana`, New("test:").Where(Where("banana", Matches, "banana"))) + testParsing(t, `query test: where banana is true`, New("test:").Where(Where("banana", Is, true))) + testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil))) + + // special + testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil)))) +} + +func testParseError(t *testing.T, queryText string, expectedErrorString string) { + _, err := ParseQuery(queryText) + if err == nil { + t.Errorf("should fail to parse: %s", queryText) + return + } + if err.Error() != expectedErrorString { + t.Errorf("unexpected error for query: %s\nwanted: %s\n got: %s", queryText, expectedErrorString, err) + } +} + +func TestParseErrors(t *testing.T) { + // syntax + testParseError(t, `query`, `unexpected end at position 5`) + testParseError(t, `query test`, `invalid prefix: test`) + testParseError(t, `query test: where`, `unexpected end at position 17`) + testParseError(t, `query test: where (`, `unexpected end at position 19`) + testParseError(t, `query test: where )`, `unknown clause ")" at position 19`) + testParseError(t, `query test: where not`, `unexpected end at position 21`) + testParseError(t, `query test: where banana`, `unexpected end at position 24`) + testParseError(t, `query test: where banana >`, `unexpected end at position 26`) + testParseError(t, `query test: where banana nope`, `unknown operator at position 26`) + testParseError(t, `query test: where banana exists or`, `unexpected end at position 34`) + testParseError(t, `query test: where banana exists and`, `unexpected end at position 35`) + testParseError(t, `query test: where banana exists and (`, `unexpected end at position 37`) + testParseError(t, `query test: where banana exists and banana is true or`, `you may not mix "and" and "or" (position: 52)`) + testParseError(t, `query test: where banana exists or banana is true and`, `you may not mix "and" and "or" (position: 51)`) + // testParseError(t, `query test: where banana exists and (`, ``) + + // value parsing error + testParseError(t, `query test: where banana == banana`, `could not parse banana to int64: strconv.ParseInt: parsing "banana": invalid syntax (hint: use "sameas" to compare strings)`) + testParseError(t, `query test: where banana f== banana`, `could not parse banana to float64: strconv.ParseFloat: parsing "banana": invalid syntax`) + testParseError(t, `query test: where banana in banana`, `could not parse "banana" to []string`) + testParseError(t, `query test: where banana matches [banana`, "could not compile regex \"[banana\": error parsing regexp: missing closing ]: `[banana`") + testParseError(t, `query test: where banana is great`, `could not parse "great" to bool: strconv.ParseBool: parsing "great": invalid syntax`) +} diff --git a/database/query/query.go b/database/query/query.go index c8fbf1e..8b4433e 100644 --- a/database/query/query.go +++ b/database/query/query.go @@ -22,56 +22,113 @@ var ( // Query contains a compiled query. type Query struct { - prefix string - condition Condition + checked bool + prefix string + where Condition + orderBy string + limit int + offset int } -// New creates a new query. -func New(prefix string, condition Condition) (*Query, error) { +// New creates a new query with the supplied prefix. +func New(prefix string) *Query { + return &Query{ + prefix: prefix, + } +} + +// Where adds filtering. +func (q *Query) Where(condition Condition) *Query { + q.where = condition + return q +} + +// Limit limits the number of returned results. +func (q *Query) Limit(limit int) *Query { + q.limit = limit + return q +} + +// Offset sets the query offset. +func (q *Query) Offset(offset int) *Query { + q.offset = offset + return q +} + +// OrderBy orders the results by the given key. +func (q *Query) OrderBy(key string) *Query { + q.orderBy = key + return q +} + +// Check checks for errors in the query. +func (q *Query) Check() (*Query, error) { + if q.checked { + return q, nil + } + // check prefix - if !prefixExpr.MatchString(prefix) { - return nil, fmt.Errorf("invalid prefix: %s", prefix) + if !prefixExpr.MatchString(q.prefix) { + return nil, fmt.Errorf("invalid prefix: %s", q.prefix) } // check condition - if condition != nil { - err := condition.check() + if q.where != nil { + err := q.where.check() if err != nil { return nil, err } } else { - condition = &noCond{} + q.where = &noCond{} } - // return query - return &Query{ - prefix: prefix, - condition: condition, - }, nil + q.checked = true + return q, nil } -// MustCompile creates a new query and panics on an error. -func MustCompile(prefix string, condition Condition) *Query { - q, err := New(prefix, condition) +// MustBeValid checks for errors in the query and panics if there is an error. +func (q *Query) MustBeValid() *Query { + _, err := q.Check() if err != nil { panic(err) } return q } -// Matches checks whether the query matches the supplied data object. -func (q *Query) Matches(f Fetcher) bool { - return q.condition.complies(f) +// IsChecked returns whether they query was checked. +func (q *Query) IsChecked() bool { + return q.checked } -// String returns the string representation of the query. -func (q *Query) String() string { - text := q.condition.string() - if text == "" { - return fmt.Sprintf("query %s", q.prefix) - } - if strings.HasPrefix(text, "(") { - text = text[1 : len(text)-1] - } - return fmt.Sprintf("query %s where %s", q.prefix, text) +// Matches checks whether the query matches the supplied data object. +func (q *Query) Matches(f Fetcher) bool { + return q.where.complies(f) +} + +// Print returns the string representation of the query. +func (q *Query) Print() string { + where := q.where.string() + if where != "" { + if strings.HasPrefix(where, "(") { + where = where[1 : len(where)-1] + } + where = fmt.Sprintf(" where %s", where) + } + + var orderBy string + if q.orderBy != "" { + orderBy = fmt.Sprintf(" orderby %s", q.orderBy) + } + + var limit string + if q.limit > 0 { + limit = fmt.Sprintf(" limit %d", q.limit) + } + + var offset string + if q.offset > 0 { + offset = fmt.Sprintf(" offset %d", q.offset) + } + + return fmt.Sprintf("query %s%s%s%s%s", q.prefix, where, orderBy, limit, offset) } diff --git a/database/query/query_test.go b/database/query/query_test.go index 39ce970..bf25e70 100644 --- a/database/query/query_test.go +++ b/database/query/query_test.go @@ -6,7 +6,7 @@ import ( var ( // copied from https://github.com/tidwall/gjson/blob/master/gjson_test.go - testJson = `{"age":100, "name":{"here":"B\\\"R"}, + testJSON = `{"age":100, "name":{"here":"B\\\"R"}, "noop":{"what is a wren?":"a bird"}, "happy":true,"immortal":false, "items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7], @@ -45,25 +45,25 @@ var ( ) func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) { - q := MustCompile("test:", condition) + q := New("test:").Where(condition).MustBeValid() // fmt.Printf("%s\n", q.String()) matched := q.Matches(f) switch { case !matched && shouldMatch: - t.Errorf("should match: %s", q.String()) + t.Errorf("should match: %s", q.Print()) case matched && !shouldMatch: - t.Errorf("should not match: %s", q.String()) + t.Errorf("should not match: %s", q.Print()) } } func TestQuery(t *testing.T) { - // if !gjson.Valid(testJson) { + // if !gjson.Valid(testJSON) { // t.Fatal("test json is invalid") // } - f := NewJSONFetcher(testJson) + f := NewJSONFetcher(testJSON) testQuery(t, f, true, Where("age", Equals, 100)) testQuery(t, f, true, Where("age", GreaterThan, uint8(99))) @@ -88,6 +88,7 @@ func TestQuery(t *testing.T) { testQuery(t, f, true, Where("lastly.yay", In, "draft,final")) testQuery(t, f, true, Where("lastly.yay", In, "final,draft")) + testQuery(t, f, true, Where("happy", Is, true)) testQuery(t, f, true, Where("happy", Is, "true")) testQuery(t, f, true, Where("happy", Is, "t")) testQuery(t, f, true, Not(Where("happy", Is, "0")))