Finish query package for now

database/query/README.md

@@ -0,0 +1,55 @@
+# Query
+
+## Control Flow
+
+- Grouping with `(` and `)`
+- Chaining with `and` and `or`
+  - _NO_ mixing! Be explicit and use grouping.
+- Negation with `not`
+  - in front of expression for group: `not (...)`
+  - inside expression for clause: `name not matches "^King "`
+
+## Selectors
+
+Supported by all feeders:
+- root level field: `field`
+- sub level field: `field.sub`
+- array/slice/map access: `map.0`
+- array/slice/map length: `map.#`
+
+Please note that some feeders may have other special characters. It is advised to only use alphanumeric characters for keys.
+
+## Operators
+
+| Name | Textual | Req. Type | Internal Type | Compared with |
+|---|---|---|---|
+| Equals | `==` | int | int64 | `==` |
+| GreaterThan | `>` | int | int64 | `>` |
+| GreaterThanOrEqual | `>=` | int | int64 | `>=` |
+| LessThan | `<`  | int | int64 | `<` |
+| LessThanOrEqual | `<=` | int | int64 | `<=` |
+| FloatEquals | `f==` | float | float64 | `==` |
+| FloatGreaterThan | `f>` | float | float64 | `>` |
+| FloatGreaterThanOrEqual | `f>=` | float | float64 | `>=` |
+| FloatLessThan | `f<` | float | float64 | `<` |
+| FloatLessThanOrEqual | `f<=` | float | float64 | `<=` |
+| SameAs | `sameas`, `s==` | string | string | `==` |
+| Contains | `contains`, `co` | string | string | `strings.Contains()` |
+| StartsWith | `startswith`, `sw` | string | string | `strings.HasPrefix()` |
+| EndsWith | `endswith`, `ew` | string | string | `strings.HasSuffix()` |
+| In | `in` | string | string | for loop with `==` |
+| Matches | `matches`, `re` | string | int64 | `regexp.Regexp.Matches()` |
+| Is | `is` | bool* | bool | `==` |
+| Exists | `exists`, `ex` | any | n/a | n/a |
+
+\*accepts strings: 1, t, T, TRUE, true, True, 0, f, F, FALSE
+
+## Escaping
+
+If you need to use a control character within a value (ie. not for controlling), escape it with `\`.
+It is recommended to wrap a word into parenthesis instead of escaping control characters, when possible.
+
+| Location | Characters to be escaped |
+|---|---|
+| Within parenthesis (`"`) | `"`, `\` |
+| Everywhere else | `(`, `)`, `"`, `\`, `\t`, `\r`, `\n`, ` ` (space) |

database/query/condition-bool.go

@@ -64,5 +64,5 @@ func (c *boolCondition) check() error {
 }
 
 func (c *boolCondition) string() string {
-	return fmt.Sprintf("%s %s %t", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %t", escapeString(c.key), getOpName(c.operator), c.value)
 }

database/query/condition-exists.go

@@ -29,5 +29,5 @@ func (c *existsCondition) check() error {
 }
 
 func (c *existsCondition) string() string {
-	return fmt.Sprintf("%s %s", c.key, getOpName(c.operator))
+	return fmt.Sprintf("%s %s", escapeString(c.key), getOpName(c.operator))
 }

database/query/condition-float.go

@@ -31,6 +31,8 @@ func newFloatCondition(key string, operator uint8, value interface{}) *floatCond
 		parsedValue = float64(v)
 	case uint8:
 		parsedValue = float64(v)
+	case uint16:
+		parsedValue = float64(v)
 	case uint32:
 		parsedValue = float64(v)
 	case float32:
@@ -90,5 +92,5 @@ func (c *floatCondition) check() error {
 }
 
 func (c *floatCondition) string() string {
-	return fmt.Sprintf("%s %s %f", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %g", escapeString(c.key), getOpName(c.operator), c.value)
 }

database/query/condition-int.go

@@ -31,6 +31,8 @@ func newIntCondition(key string, operator uint8, value interface{}) *intConditio
 		parsedValue = int64(v)
 	case uint8:
 		parsedValue = int64(v)
+	case uint16:
+		parsedValue = int64(v)
 	case uint32:
 		parsedValue = int64(v)
 	case string:
@@ -86,5 +88,5 @@ func (c *intCondition) check() error {
 }
 
 func (c *intCondition) string() string {
-	return fmt.Sprintf("%s %s %d", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %d", escapeString(c.key), getOpName(c.operator), c.value)
 }

database/query/condition-regex.go

@@ -57,5 +57,5 @@ func (c *regexCondition) check() error {
 }
 
 func (c *regexCondition) string() string {
-	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.regex.String())
+	return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.regex.String()))
 }

database/query/condition-string.go

@@ -56,5 +56,5 @@ func (c *stringCondition) check() error {
 }
 
 func (c *stringCondition) string() string {
-	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), c.value)
+	return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(c.value))
 }

database/query/condition-stringslice.go

@@ -20,7 +20,7 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
 		parsedValue := strings.Split(v, ",")
 		if len(parsedValue) < 2 {
 			return &stringSliceCondition{
-				key:      fmt.Sprintf("could not parse \"%s\" to []string", v),
+				key:      v,
 				operator: errorPresent,
 			}
 		}
@@ -29,6 +29,12 @@ func newStringSliceCondition(key string, operator uint8, value interface{}) *str
 			operator: operator,
 			value:    parsedValue,
 		}
+	case []string:
+		return &stringSliceCondition{
+			key:      key,
+			operator: operator,
+			value:    v,
+		}
 	default:
 		return &stringSliceCondition{
 			key:      fmt.Sprintf("incompatible value %v for []string", value),
@@ -60,5 +66,5 @@ func (c *stringSliceCondition) check() error {
 }
 
 func (c *stringSliceCondition) string() string {
-	return fmt.Sprintf("%s %s %s", c.key, getOpName(c.operator), strings.Join(c.value, ","))
+	return fmt.Sprintf("%s %s %s", escapeString(c.key), getOpName(c.operator), escapeString(strings.Join(c.value, ",")))
 }

database/query/condition_test.go

@@ -0,0 +1,76 @@
+package query
+
+import "testing"
+
+func testSuccess(t *testing.T, c Condition) {
+	err := c.check()
+	if err != nil {
+		t.Errorf("failed: %s", err)
+	}
+}
+
+func TestInterfaces(t *testing.T) {
+	testSuccess(t, newIntCondition("banana", Equals, uint(1)))
+	testSuccess(t, newIntCondition("banana", Equals, uint8(1)))
+	testSuccess(t, newIntCondition("banana", Equals, uint16(1)))
+	testSuccess(t, newIntCondition("banana", Equals, uint32(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int8(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int16(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int32(1)))
+	testSuccess(t, newIntCondition("banana", Equals, int64(1)))
+	testSuccess(t, newIntCondition("banana", Equals, "1"))
+
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint8(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint16(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, uint32(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int8(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int16(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int32(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, int64(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, float32(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, float64(1)))
+	testSuccess(t, newFloatCondition("banana", FloatEquals, "1.1"))
+
+	testSuccess(t, newStringCondition("banana", SameAs, "coconut"))
+	testSuccess(t, newRegexCondition("banana", Matches, "coconut"))
+	testSuccess(t, newStringSliceCondition("banana", FloatEquals, []string{"banana", "coconut"}))
+	testSuccess(t, newStringSliceCondition("banana", FloatEquals, "banana,coconut"))
+}
+
+func testCondError(t *testing.T, c Condition) {
+	err := c.check()
+	if err == nil {
+		t.Error("should fail")
+	}
+}
+
+func TestConditionErrors(t *testing.T) {
+	// test invalid value types
+	testCondError(t, newBoolCondition("banana", Is, 1))
+	testCondError(t, newFloatCondition("banana", FloatEquals, true))
+	testCondError(t, newIntCondition("banana", Equals, true))
+	testCondError(t, newStringCondition("banana", SameAs, 1))
+	testCondError(t, newRegexCondition("banana", Matches, 1))
+	testCondError(t, newStringSliceCondition("banana", Matches, 1))
+
+	// test error presence
+	testCondError(t, newBoolCondition("banana", errorPresent, true))
+	testCondError(t, And(newBoolCondition("banana", errorPresent, true)))
+	testCondError(t, Or(newBoolCondition("banana", errorPresent, true)))
+	testCondError(t, newExistsCondition("banana", errorPresent))
+	testCondError(t, newFloatCondition("banana", errorPresent, 1.1))
+	testCondError(t, newIntCondition("banana", errorPresent, 1))
+	testCondError(t, newStringCondition("banana", errorPresent, "coconut"))
+	testCondError(t, newRegexCondition("banana", errorPresent, "coconut"))
+}
+
+func TestWhere(t *testing.T) {
+	c := Where("", 254, nil)
+	err := c.check()
+	if err == nil {
+		t.Error("should fail")
+	}
+}

database/query/operators.go

@@ -0,0 +1,53 @@
+package query
+
+var (
+	operatorNames = map[string]uint8{
+		"==":         Equals,
+		">":          GreaterThan,
+		">=":         GreaterThanOrEqual,
+		"<":          LessThan,
+		"<=":         LessThanOrEqual,
+		"f==":        FloatEquals,
+		"f>":         FloatGreaterThan,
+		"f>=":        FloatGreaterThanOrEqual,
+		"f<":         FloatLessThan,
+		"f<=":        FloatLessThanOrEqual,
+		"sameas":     SameAs,
+		"s==":        SameAs,
+		"contains":   Contains,
+		"co":         Contains,
+		"startswith": StartsWith,
+		"sw":         StartsWith,
+		"endswith":   EndsWith,
+		"ew":         EndsWith,
+		"in":         In,
+		"matches":    Matches,
+		"re":         Matches,
+		"is":         Is,
+		"exists":     Exists,
+		"ex":         Exists,
+	}
+
+	primaryNames = make(map[uint8]string)
+)
+
+func init() {
+	for opName, opID := range operatorNames {
+		name, ok := primaryNames[opID]
+		if ok {
+			if len(name) < len(opName) {
+				primaryNames[opID] = opName
+			}
+		} else {
+			primaryNames[opID] = opName
+		}
+	}
+}
+
+func getOpName(operator uint8) string {
+	name, ok := primaryNames[operator]
+	if ok {
+		return name
+	}
+	return "[unknown]"
+}

database/query/operators_test.go

@@ -0,0 +1,9 @@
+package query
+
+import "testing"
+
+func TestGetOpName(t *testing.T) {
+	if getOpName(254) != "[unknown]" {
+		t.Error("unexpected output")
+	}
+}

database/query/parser.go

@@ -4,77 +4,122 @@ import (
 	"errors"
 	"fmt"
 	"regexp"
+	"strconv"
 	"strings"
 )
 
-var (
+type snippet struct {
-	operatorNames = map[string]uint8{
-		"==":         Equals,
-		">":          GreaterThan,
-		">=":         GreaterThanOrEqual,
-		"<":          LessThan,
-		"<=":         LessThanOrEqual,
-		"f==":        FloatEquals,
-		"f>":         FloatGreaterThan,
-		"f>=":        FloatGreaterThanOrEqual,
-		"f<":         FloatLessThan,
-		"f<=":        FloatLessThanOrEqual,
-		"sameas":     SameAs,
-		"s==":        SameAs,
-		"contains":   Contains,
-		"co":         Contains,
-		"startswith": StartsWith,
-		"sw":         StartsWith,
-		"endswith":   EndsWith,
-		"ew":         EndsWith,
-		"in":         In,
-		"matches":    Matches,
-		"re":         Matches,
-		"is":         Is,
-		"exists":     Exists,
-		"ex":         Exists,
-	}
-
-	primaryNames = make(map[uint8]string)
-)
-
-func init() {
-	for opName, opID := range operatorNames {
-		name, ok := primaryNames[opID]
-		if ok {
-			if len(name) < len(opName) {
-				primaryNames[opID] = opName
-			}
-		} else {
-			primaryNames[opID] = opName
-		}
-	}
-}
-
-func getOpName(operator uint8) string {
-	name, ok := primaryNames[operator]
-	if ok {
-		return name
-	}
-	return "[unknown]"
-}
-
-type treeElement struct {
-	branches       []*treeElement
 	text           string
 	globalPosition int
 }
 
-var (
+// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
-	escapeReplacer = regexp.MustCompile("\\\\([^\\\\])")
+func ParseQuery(query string) (*Query, error) {
-)
+	snippets, err := extractSnippets(query)
+	if err != nil {
+		return nil, err
+	}
+	snippetsPos := 0
 
-// prepToken removes surrounding parenthesis and escape characters.
+	getSnippet := func() (*snippet, error) {
-func prepToken(text string) string {
+		// order is important, as parseAndOr will always consume one additional snippet.
-	return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
+		snippetsPos++
+		if snippetsPos > len(snippets) {
+			return nil, fmt.Errorf("unexpected end at position %d", len(query))
+		}
+		return snippets[snippetsPos-1], nil
+	}
+	remainingSnippets := func() int {
+		return len(snippets) - snippetsPos
+	}
+
+	// check for query word
+	queryWord, err := getSnippet()
+	if err != nil {
+		return nil, err
+	}
+	if queryWord.text != "query" {
+		return nil, errors.New("queries must start with \"query\"")
+	}
+
+	// get prefix
+	prefix, err := getSnippet()
+	if err != nil {
+		return nil, err
+	}
+	q := New(prefix.text)
+
+	for remainingSnippets() > 0 {
+		command, err := getSnippet()
+		if err != nil {
+			return nil, err
+		}
+
+		switch command.text {
+		case "where":
+			if q.where != nil {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			// parse conditions
+			condition, err := parseAndOr(getSnippet, remainingSnippets, true)
+			if err != nil {
+				return nil, err
+			}
+			// go one back, as parseAndOr had to check if its done
+			snippetsPos--
+
+			q.Where(condition)
+		case "orderby":
+			if q.orderBy != "" {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			orderBySnippet, err := getSnippet()
+			if err != nil {
+				return nil, err
+			}
+
+			q.OrderBy(orderBySnippet.text)
+		case "limit":
+			if q.limit != 0 {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			limitSnippet, err := getSnippet()
+			if err != nil {
+				return nil, err
+			}
+			limit, err := strconv.ParseUint(limitSnippet.text, 10, 31)
+			if err != nil {
+				return nil, fmt.Errorf("could not parse integer (%s) at position %d", limitSnippet.text, limitSnippet.globalPosition)
+			}
+
+			q.Limit(int(limit))
+		case "offset":
+			if q.offset != 0 {
+				return nil, fmt.Errorf("duplicate \"%s\" clause found at position %d", command.text, command.globalPosition)
+			}
+
+			offsetSnippet, err := getSnippet()
+			if err != nil {
+				return nil, err
+			}
+			offset, err := strconv.ParseUint(offsetSnippet.text, 10, 31)
+			if err != nil {
+				return nil, fmt.Errorf("could not parse integer (%s) at position %d", offsetSnippet.text, offsetSnippet.globalPosition)
+			}
+
+			q.Offset(int(offset))
+		default:
+			return nil, fmt.Errorf("unknown clause \"%s\" at position %d", command.text, command.globalPosition)
+		}
+	}
+
+	return q.Check()
 }
 
-func extractSnippets(text string) (snippets []*treeElement, err error) {
+func extractSnippets(text string) (snippets []*snippet, err error) {
 
 	skip := false
 	start := -1
@@ -93,10 +138,10 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 			skip = true
 		}
 
-		// wait for parenthesis to be over
+		// wait for parenthesis to be overs
 		if inParenthesis {
 			if char == '"' {
-				snippets = append(snippets, &treeElement{
+				snippets = append(snippets, &snippet{
 					text:           prepToken(text[start+1 : pos]),
 					globalPosition: start + 1,
 				})
@@ -110,9 +155,9 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 		switch char {
 		case '\t', '\n', '\r', ' ', '(', ')':
 			if start >= 0 {
-				snippets = append(snippets, &treeElement{
+				snippets = append(snippets, &snippet{
 					text:           prepToken(text[start:pos]),
-					globalPosition: start,
+					globalPosition: start + 1,
 				})
 				start = -1
 			}
@@ -125,13 +170,13 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 		// handle special segment characters
 		switch char {
 		case '(', ')':
-			snippets = append(snippets, &treeElement{
+			snippets = append(snippets, &snippet{
 				text:           text[pos : pos+1],
-				globalPosition: pos,
+				globalPosition: pos + 1,
 			})
 		case '"':
 			if start < pos {
-				return nil, fmt.Errorf("parenthesis ('\"') may not be within words, please escape with '\\' (position: %d)", pos+1)
+				return nil, fmt.Errorf("parenthesis ('\"') may not be used within words, please escape with '\\' (position: %d)", pos+1)
 			}
 			inParenthesis = true
 		}
@@ -139,89 +184,166 @@ func extractSnippets(text string) (snippets []*treeElement, err error) {
 	}
 
 	// add last
-	snippets = append(snippets, &treeElement{
+	if start >= 0 {
-		text:           prepToken(text[start : pos+1]),
+		snippets = append(snippets, &snippet{
-		globalPosition: start,
+			text:           prepToken(text[start : pos+1]),
-	})
+			globalPosition: start + 1,
+		})
+	}
 
 	return snippets, nil
 
 }
 
-// ParseQuery parses a plaintext query. Special characters (that must be escaped with a '\') are: `\()` and any whitespaces.
+func parseAndOr(getSnippet func() (*snippet, error), remainingSnippets func() int, rootCondition bool) (Condition, error) {
-func ParseQuery(query string) (*Query, error) {
+	var isOr = false
-	snippets, err := extractSnippets(query)
+	var typeSet = false
-	if err != nil {
+	var wrapInNot = false
-		return nil, err
+	var expectingMore = true
-	}
+	var conditions []Condition
-	snippetsPos := 0
 
-	getElement := func() (*treeElement, error) {
+	for {
-		if snippetsPos >= len(snippets) {
+		if !expectingMore && rootCondition && remainingSnippets() == 0 {
-			return nil, fmt.Errorf("unexpected end at position %d", len(query))
+			// advance snippetsPos by one, as it will be set back by 1
+			getSnippet()
+			if len(conditions) == 1 {
+				return conditions[0], nil
+			}
+			if isOr {
+				return Or(conditions...), nil
+			}
+			return And(conditions...), nil
 		}
-		return snippets[snippetsPos], nil
-	}
 
-	// check for query word
+		firstSnippet, err := getSnippet()
-	queryWord, err := getElement()
+		if err != nil {
-	if err != nil {
+			return nil, err
-		return nil, err
+		}
-	}
-	if queryWord.text != "query" {
-		return nil, errors.New("queries must start with \"query\"")
-	}
 
-	// get prefix
+		if !expectingMore && rootCondition {
-	prefix, err := getElement()
+			switch firstSnippet.text {
-	if err != nil {
+			case "orderby", "limit", "offset":
-		return nil, err
+				if len(conditions) == 1 {
-	}
+					return conditions[0], nil
+				}
+				if isOr {
+					return Or(conditions...), nil
+				}
+				return And(conditions...), nil
+			}
+		}
 
-	// check if no condition
+		switch firstSnippet.text {
-	if len(snippets) == 2 {
+		case "(":
-		return New(prefix.text, nil)
+			condition, err := parseAndOr(getSnippet, remainingSnippets, false)
+			if err != nil {
+				return nil, err
+			}
+			if wrapInNot {
+				conditions = append(conditions, Not(condition))
+				wrapInNot = false
+			} else {
+				conditions = append(conditions, condition)
+			}
+			expectingMore = true
+		case ")":
+			if len(conditions) == 1 {
+				return conditions[0], nil
+			}
+			if isOr {
+				return Or(conditions...), nil
+			}
+			return And(conditions...), nil
+		case "and":
+			if typeSet && isOr {
+				return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
+			}
+			isOr = false
+			typeSet = true
+			expectingMore = true
+		case "or":
+			if typeSet && !isOr {
+				return nil, fmt.Errorf("you may not mix \"and\" and \"or\" (position: %d)", firstSnippet.globalPosition)
+			}
+			isOr = true
+			typeSet = true
+			expectingMore = true
+		case "not":
+			wrapInNot = true
+			expectingMore = true
+		default:
+			condition, err := parseCondition(firstSnippet, getSnippet)
+			if err != nil {
+				return nil, err
+			}
+			if wrapInNot {
+				conditions = append(conditions, Not(condition))
+				wrapInNot = false
+			} else {
+				conditions = append(conditions, condition)
+			}
+			expectingMore = false
+		}
 	}
-
-	// check for where word
-	whereWord, err := getElement()
-	if err != nil {
-		return nil, err
-	}
-	if whereWord.text != "where" {
-		return nil, errors.New("filtering queries must start conditions with \"where\"")
-	}
-
-	// parse conditions
-	condition, err := parseCondition(getElement)
-	if err != nil {
-		return nil, err
-	}
-
-	// check for additional tokens
-	// token := s.Scan()
-	// if token != scanner.EOF {
-	// 	return nil, fmt.Errorf("unexpected additional tokens at position %d", s.Position)
-	// }
-
-	return New(prefix.text, condition)
 }
 
-func parseCondition(getElement func() (*treeElement, error)) (Condition, error) {
+func parseCondition(firstSnippet *snippet, getSnippet func() (*snippet, error)) (Condition, error) {
-	first, err := getElement()
+	wrapInNot := false
+
+	// get operator name
+	opName, err := getSnippet()
 	if err != nil {
 		return nil, err
 	}
-
+	// negate?
-	switch first.text {
+	if opName.text == "not" {
-	case "(":
+		wrapInNot = true
-		return parseAndOr(getElement, true, nil, false)
+		opName, err = getSnippet()
-		// case ""
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	return nil, nil
+	// get operator
+	operator, ok := operatorNames[opName.text]
+	if !ok {
+		return nil, fmt.Errorf("unknown operator at position %d", opName.globalPosition)
+	}
+
+	// don't need a value for "exists"
+	if operator == Exists {
+		if wrapInNot {
+			return Not(Where(firstSnippet.text, operator, nil)), nil
+		}
+		return Where(firstSnippet.text, operator, nil), nil
+	}
+
+	// get value
+	value, err := getSnippet()
+	if err != nil {
+		return nil, err
+	}
+	if wrapInNot {
+		return Not(Where(firstSnippet.text, operator, value.text)), nil
+	}
+	return Where(firstSnippet.text, operator, value.text), nil
 }
 
-func parseAndOr(getElement func() (*treeElement, error), expectBracket bool, preParsedCondition Condition, preParsedIsOr bool) (Condition, error) {
+var (
-	return nil, nil
+	escapeReplacer = regexp.MustCompile("\\\\([^\\\\])")
+)
+
+// prepToken removes surrounding parenthesis and escape characters.
+func prepToken(text string) string {
+	return escapeReplacer.ReplaceAllString(strings.Trim(text, "\""), "$1")
+}
+
+// escapeString correctly escapes a snippet for printing
+func escapeString(token string) string {
+	// check if token contains characters that need to be escaped
+	if strings.ContainsAny(token, "()\"\\\t\r\n ") {
+		// put the token in parenthesis and only escape \ and "
+		return fmt.Sprintf("\"%s\"", strings.Replace(token, "\"", "\\\"", -1))
+	}
+	return token
 }

database/query/parser_test.go

@@ -1,18 +1,168 @@
 package query
 
 import (
+	"reflect"
 	"testing"
+
+	"github.com/davecgh/go-spew/spew"
 )
 
-func TestMakeTree(t *testing.T) {
+func TestExtractSnippets(t *testing.T) {
-	text1 := `(age > 100 and experience <= "99") or (age < 10 and motivation > 50) or name matches Mr.\"X or name matches y`
+	text1 := `query test: where ( "bananas" > 100 and monkeys.# <= "12")or(coconuts < 10 "and" area > 50) or name sameas Julian or name matches ^King\ `
+	result1 := []*snippet{
+		&snippet{text: "query", globalPosition: 1},
+		&snippet{text: "test:", globalPosition: 7},
+		&snippet{text: "where", globalPosition: 13},
+		&snippet{text: "(", globalPosition: 19},
+		&snippet{text: "bananas", globalPosition: 21},
+		&snippet{text: ">", globalPosition: 31},
+		&snippet{text: "100", globalPosition: 33},
+		&snippet{text: "and", globalPosition: 37},
+		&snippet{text: "monkeys.#", globalPosition: 41},
+		&snippet{text: "<=", globalPosition: 51},
+		&snippet{text: "12", globalPosition: 54},
+		&snippet{text: ")", globalPosition: 58},
+		&snippet{text: "or", globalPosition: 59},
+		&snippet{text: "(", globalPosition: 61},
+		&snippet{text: "coconuts", globalPosition: 62},
+		&snippet{text: "<", globalPosition: 71},
+		&snippet{text: "10", globalPosition: 73},
+		&snippet{text: "and", globalPosition: 76},
+		&snippet{text: "area", globalPosition: 82},
+		&snippet{text: ">", globalPosition: 87},
+		&snippet{text: "50", globalPosition: 89},
+		&snippet{text: ")", globalPosition: 91},
+		&snippet{text: "or", globalPosition: 93},
+		&snippet{text: "name", globalPosition: 96},
+		&snippet{text: "sameas", globalPosition: 101},
+		&snippet{text: "Julian", globalPosition: 108},
+		&snippet{text: "or", globalPosition: 115},
+		&snippet{text: "name", globalPosition: 118},
+		&snippet{text: "matches", globalPosition: 123},
+		&snippet{text: "^King ", globalPosition: 131},
+	}
+
 	snippets, err := extractSnippets(text1)
 	if err != nil {
-		t.Errorf("failed to make tree: %s", err)
+		t.Errorf("failed to extract snippets: %s", err)
-	} else {
+	}
+
+	if !reflect.DeepEqual(result1, snippets) {
+		t.Errorf("unexpected results:")
 		for _, el := range snippets {
 			t.Errorf("%+v", el)
 		}
 	}
+
 	// t.Error(spew.Sprintf("%v", treeElement))
 }
+
+func testParsing(t *testing.T, queryText string, expectedResult *Query) {
+	_, err := expectedResult.Check()
+	if err != nil {
+		t.Errorf("failed to create query: %s", err)
+		return
+	}
+
+	q, err := ParseQuery(queryText)
+	if err != nil {
+		t.Errorf("failed to parse query: %s", err)
+		return
+	}
+
+	if queryText != q.Print() {
+		t.Errorf("string match failed: %s", q.Print())
+		return
+	}
+	if !reflect.DeepEqual(expectedResult, q) {
+		t.Error("deepqual match failed.")
+		t.Error("got:")
+		t.Error(spew.Sdump(q))
+		t.Error("expected:")
+		t.Error(spew.Sdump(expectedResult))
+	}
+}
+
+func TestParseQuery(t *testing.T) {
+	text1 := `query test: where (bananas > 100 and monkeys.# <= 12) or not (coconuts < 10 and area not > 50) or name sameas Julian or name matches "^King " orderby name limit 10 offset 20`
+	result1 := New("test:").Where(Or(
+		And(
+			Where("bananas", GreaterThan, 100),
+			Where("monkeys.#", LessThanOrEqual, 12),
+		),
+		Not(And(
+			Where("coconuts", LessThan, 10),
+			Not(Where("area", GreaterThan, 50)),
+		)),
+		Where("name", SameAs, "Julian"),
+		Where("name", Matches, "^King "),
+	)).OrderBy("name").Limit(10).Offset(20)
+	testParsing(t, text1, result1)
+
+	testParsing(t, `query test: orderby name`, New("test:").OrderBy("name"))
+	testParsing(t, `query test: limit 10`, New("test:").Limit(10))
+	testParsing(t, `query test: offset 10`, New("test:").Offset(10))
+	testParsing(t, `query test: where banana matches ^ban`, New("test:").Where(Where("banana", Matches, "^ban")))
+	testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
+	testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
+
+	// test all operators
+	testParsing(t, `query test: where banana == 1`, New("test:").Where(Where("banana", Equals, 1)))
+	testParsing(t, `query test: where banana > 1`, New("test:").Where(Where("banana", GreaterThan, 1)))
+	testParsing(t, `query test: where banana >= 1`, New("test:").Where(Where("banana", GreaterThanOrEqual, 1)))
+	testParsing(t, `query test: where banana < 1`, New("test:").Where(Where("banana", LessThan, 1)))
+	testParsing(t, `query test: where banana <= 1`, New("test:").Where(Where("banana", LessThanOrEqual, 1)))
+	testParsing(t, `query test: where banana f== 1.1`, New("test:").Where(Where("banana", FloatEquals, 1.1)))
+	testParsing(t, `query test: where banana f> 1.1`, New("test:").Where(Where("banana", FloatGreaterThan, 1.1)))
+	testParsing(t, `query test: where banana f>= 1.1`, New("test:").Where(Where("banana", FloatGreaterThanOrEqual, 1.1)))
+	testParsing(t, `query test: where banana f< 1.1`, New("test:").Where(Where("banana", FloatLessThan, 1.1)))
+	testParsing(t, `query test: where banana f<= 1.1`, New("test:").Where(Where("banana", FloatLessThanOrEqual, 1.1)))
+	testParsing(t, `query test: where banana sameas banana`, New("test:").Where(Where("banana", SameAs, "banana")))
+	testParsing(t, `query test: where banana contains banana`, New("test:").Where(Where("banana", Contains, "banana")))
+	testParsing(t, `query test: where banana startswith banana`, New("test:").Where(Where("banana", StartsWith, "banana")))
+	testParsing(t, `query test: where banana endswith banana`, New("test:").Where(Where("banana", EndsWith, "banana")))
+	testParsing(t, `query test: where banana in banana,coconut`, New("test:").Where(Where("banana", In, []string{"banana", "coconut"})))
+	testParsing(t, `query test: where banana matches banana`, New("test:").Where(Where("banana", Matches, "banana")))
+	testParsing(t, `query test: where banana is true`, New("test:").Where(Where("banana", Is, true)))
+	testParsing(t, `query test: where banana exists`, New("test:").Where(Where("banana", Exists, nil)))
+
+	// special
+	testParsing(t, `query test: where banana not exists`, New("test:").Where(Not(Where("banana", Exists, nil))))
+}
+
+func testParseError(t *testing.T, queryText string, expectedErrorString string) {
+	_, err := ParseQuery(queryText)
+	if err == nil {
+		t.Errorf("should fail to parse: %s", queryText)
+		return
+	}
+	if err.Error() != expectedErrorString {
+		t.Errorf("unexpected error for query: %s\nwanted: %s\n   got: %s", queryText, expectedErrorString, err)
+	}
+}
+
+func TestParseErrors(t *testing.T) {
+	// syntax
+	testParseError(t, `query`, `unexpected end at position 5`)
+	testParseError(t, `query test`, `invalid prefix: test`)
+	testParseError(t, `query test: where`, `unexpected end at position 17`)
+	testParseError(t, `query test: where (`, `unexpected end at position 19`)
+	testParseError(t, `query test: where )`, `unknown clause ")" at position 19`)
+	testParseError(t, `query test: where not`, `unexpected end at position 21`)
+	testParseError(t, `query test: where banana`, `unexpected end at position 24`)
+	testParseError(t, `query test: where banana >`, `unexpected end at position 26`)
+	testParseError(t, `query test: where banana nope`, `unknown operator at position 26`)
+	testParseError(t, `query test: where banana exists or`, `unexpected end at position 34`)
+	testParseError(t, `query test: where banana exists and`, `unexpected end at position 35`)
+	testParseError(t, `query test: where banana exists and (`, `unexpected end at position 37`)
+	testParseError(t, `query test: where banana exists and banana is true or`, `you may not mix "and" and "or" (position: 52)`)
+	testParseError(t, `query test: where banana exists or banana is true and`, `you may not mix "and" and "or" (position: 51)`)
+	// testParseError(t, `query test: where banana exists and (`, ``)
+
+	// value parsing error
+	testParseError(t, `query test: where banana == banana`, `could not parse banana to int64: strconv.ParseInt: parsing "banana": invalid syntax (hint: use "sameas" to compare strings)`)
+	testParseError(t, `query test: where banana f== banana`, `could not parse banana to float64: strconv.ParseFloat: parsing "banana": invalid syntax`)
+	testParseError(t, `query test: where banana in banana`, `could not parse "banana" to []string`)
+	testParseError(t, `query test: where banana matches [banana`, "could not compile regex \"[banana\": error parsing regexp: missing closing ]: `[banana`")
+	testParseError(t, `query test: where banana is great`, `could not parse "great" to bool: strconv.ParseBool: parsing "great": invalid syntax`)
+}

database/query/query.go

@@ -22,56 +22,113 @@ var (
 
 // Query contains a compiled query.
 type Query struct {
-	prefix    string
+	checked bool
-	condition Condition
+	prefix  string
+	where   Condition
+	orderBy string
+	limit   int
+	offset  int
 }
 
-// New creates a new query.
+// New creates a new query with the supplied prefix.
-func New(prefix string, condition Condition) (*Query, error) {
+func New(prefix string) *Query {
+	return &Query{
+		prefix: prefix,
+	}
+}
+
+// Where adds filtering.
+func (q *Query) Where(condition Condition) *Query {
+	q.where = condition
+	return q
+}
+
+// Limit limits the number of returned results.
+func (q *Query) Limit(limit int) *Query {
+	q.limit = limit
+	return q
+}
+
+// Offset sets the query offset.
+func (q *Query) Offset(offset int) *Query {
+	q.offset = offset
+	return q
+}
+
+// OrderBy orders the results by the given key.
+func (q *Query) OrderBy(key string) *Query {
+	q.orderBy = key
+	return q
+}
+
+// Check checks for errors in the query.
+func (q *Query) Check() (*Query, error) {
+	if q.checked {
+		return q, nil
+	}
+
 	// check prefix
-	if !prefixExpr.MatchString(prefix) {
+	if !prefixExpr.MatchString(q.prefix) {
-		return nil, fmt.Errorf("invalid prefix: %s", prefix)
+		return nil, fmt.Errorf("invalid prefix: %s", q.prefix)
 	}
 
 	// check condition
-	if condition != nil {
+	if q.where != nil {
-		err := condition.check()
+		err := q.where.check()
 		if err != nil {
 			return nil, err
 		}
 	} else {
-		condition = &noCond{}
+		q.where = &noCond{}
 	}
 
-	// return query
+	q.checked = true
-	return &Query{
+	return q, nil
-		prefix:    prefix,
-		condition: condition,
-	}, nil
 }
 
-// MustCompile creates a new query and panics on an error.
+// MustBeValid checks for errors in the query and panics if there is an error.
-func MustCompile(prefix string, condition Condition) *Query {
+func (q *Query) MustBeValid() *Query {
-	q, err := New(prefix, condition)
+	_, err := q.Check()
 	if err != nil {
 		panic(err)
 	}
 	return q
 }
 
-// Matches checks whether the query matches the supplied data object.
+// IsChecked returns whether they query was checked.
-func (q *Query) Matches(f Fetcher) bool {
+func (q *Query) IsChecked() bool {
-	return q.condition.complies(f)
+	return q.checked
 }
 
-// String returns the string representation of the query.
+// Matches checks whether the query matches the supplied data object.
-func (q *Query) String() string {
+func (q *Query) Matches(f Fetcher) bool {
-	text := q.condition.string()
+	return q.where.complies(f)
-	if text == "" {
+}
-		return fmt.Sprintf("query %s", q.prefix)
+
-	}
+// Print returns the string representation of the query.
-	if strings.HasPrefix(text, "(") {
+func (q *Query) Print() string {
-		text = text[1 : len(text)-1]
+	where := q.where.string()
-	}
+	if where != "" {
-	return fmt.Sprintf("query %s where %s", q.prefix, text)
+		if strings.HasPrefix(where, "(") {
+			where = where[1 : len(where)-1]
+		}
+		where = fmt.Sprintf(" where %s", where)
+	}
+
+	var orderBy string
+	if q.orderBy != "" {
+		orderBy = fmt.Sprintf(" orderby %s", q.orderBy)
+	}
+
+	var limit string
+	if q.limit > 0 {
+		limit = fmt.Sprintf(" limit %d", q.limit)
+	}
+
+	var offset string
+	if q.offset > 0 {
+		offset = fmt.Sprintf(" offset %d", q.offset)
+	}
+
+	return fmt.Sprintf("query %s%s%s%s%s", q.prefix, where, orderBy, limit, offset)
 }

database/query/query_test.go

@@ -6,7 +6,7 @@ import (
 
 var (
 	// copied from https://github.com/tidwall/gjson/blob/master/gjson_test.go
-	testJson = `{"age":100, "name":{"here":"B\\\"R"},
+	testJSON = `{"age":100, "name":{"here":"B\\\"R"},
   "noop":{"what is a wren?":"a bird"},
   "happy":true,"immortal":false,
   "items":[1,2,3,{"tags":[1,2,3],"points":[[1,2],[3,4]]},4,5,6,7],
@@ -45,25 +45,25 @@ var (
 )
 
 func testQuery(t *testing.T, f Fetcher, shouldMatch bool, condition Condition) {
-	q := MustCompile("test:", condition)
+	q := New("test:").Where(condition).MustBeValid()
 
 	// fmt.Printf("%s\n", q.String())
 
 	matched := q.Matches(f)
 	switch {
 	case !matched && shouldMatch:
-		t.Errorf("should match: %s", q.String())
+		t.Errorf("should match: %s", q.Print())
 	case matched && !shouldMatch:
-		t.Errorf("should not match: %s", q.String())
+		t.Errorf("should not match: %s", q.Print())
 	}
 }
 
 func TestQuery(t *testing.T) {
 
-	// if !gjson.Valid(testJson) {
+	// if !gjson.Valid(testJSON) {
 	// 	t.Fatal("test json is invalid")
 	// }
-	f := NewJSONFetcher(testJson)
+	f := NewJSONFetcher(testJSON)
 
 	testQuery(t, f, true, Where("age", Equals, 100))
 	testQuery(t, f, true, Where("age", GreaterThan, uint8(99)))
@@ -88,6 +88,7 @@ func TestQuery(t *testing.T) {
 	testQuery(t, f, true, Where("lastly.yay", In, "draft,final"))
 	testQuery(t, f, true, Where("lastly.yay", In, "final,draft"))
 
+	testQuery(t, f, true, Where("happy", Is, true))
 	testQuery(t, f, true, Where("happy", Is, "true"))
 	testQuery(t, f, true, Where("happy", Is, "t"))
 	testQuery(t, f, true, Not(Where("happy", Is, "0")))