eliasdb/eql/parser/parser_test.go

/*
 * EliasDB
 *
 * Copyright 2016 Matthias Ladkau. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

package parser

import (
	"bytes"
	"encoding/json"
	"fmt"
	"testing"
)

/*
Test RuntimeProvider provides runtime components for a parse tree.
*/
type TestRuntimeProvider struct {
}

/*
Runtime returns a runtime component for a given ASTNode.
*/
func (trp *TestRuntimeProvider) Runtime(node *ASTNode) Runtime {
	return &TestRuntime{}
}

/*
Test Runtime provides the runtime for an ASTNode.
*/
type TestRuntime struct {
}

/*
Validate this runtime component and all its child components.
*/
func (tr *TestRuntime) Validate() error {
	return nil
}

/*
Eval evaluate this runtime component.
*/
func (tr *TestRuntime) Eval() (interface{}, error) {
	return nil, nil
}

func TestSimpleExpressionParsing(t *testing.T) {

	// Test error output

	input := `"bl\*a"`
	if _, err := Parse("mytest", input); err.Error() !=
		"Parse error in mytest: Lexical error (invalid syntax while parsing escape sequences) (Line:1 Pos:1)" {
		t.Error(err)
		return
	}

	// Test incomplete expression

	input = `a *`
	if _, err := Parse("mytest", input); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error(err)
		return
	}

	// Test prefix operator

	input = ` + a - -5`
	expectedOutput := `
minus
  plus
    value: "a"
  minus
    value: "5"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	// Test simple arithmetics

	input = "a + b * 5 /2"
	expectedOutput = `
plus
  value: "a"
  div
    times
      value: "b"
      value: "5"
    value: "2"
`[1:]

	if res, err := ParseWithRuntime("mytest", input, &TestRuntimeProvider{}); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	// Test brackets

	input = "a + 1 * (5 + 6)"
	expectedOutput = `
plus
  value: "a"
  times
    value: "1"
    plus
      value: "5"
      value: "6"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	input = "(a + 1) * 5 / (6 - 2)"
	expectedOutput = `
div
  times
    plus
      value: "a"
      value: "1"
    value: "5"
  minus
    value: "6"
    value: "2"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	input = "a + 1 * [1,2,[1,2],3]"
	expectedOutput = `
plus
  value: "a"
  times
    value: "1"
    list
      value: "1"
      value: "2"
      list
        value: "1"
        value: "2"
      value: "3"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	// Test logical operators

	input = "not (a + 1) * 5 and tRue or not 1 - 5 != !test"
	expectedOutput = `
or
  and
    not
      times
        plus
          value: "a"
          value: "1"
        value: "5"
    true
  not
    !=
      minus
        value: "1"
        value: "5"
      value: "!test"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}
}

func TestQueryParsing(t *testing.T) {

	// Test get expressions

	input := `
GeT Song FROM group test`
	expectedOutput := `
get
  value: "Song"
  from
    group
      value: "test"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	// Test lookup expressions

	input = `
lOOkup Song "a","b","c", "blaД"
FROM group test`
	expectedOutput = `
lookup
  value: "Song"
  value: "a"
  value: "b"
  value: "c"
  value: "blaД"
  from
    group
      value: "test"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	// Test where clause

	input = `
GeT bla FROM group test where x = 1 AND b = -1 where True`
	expectedOutput = `
get
  value: "bla"
  from
    group
      value: "test"
  where
    and
      =
        value: "x"
        value: "1"
      =
        value: "b"
        minus
          value: "1"
  where
    true
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	input = `
GeT bla where nest.nint = 1 AND b = -1`
	expectedOutput = `
get
  value: "bla"
  where
    and
      =
        value: "nest.nint"
        value: "1"
      =
        value: "b"
        minus
          value: "1"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	// Test traverse clause

	input = `
GeT bla TraverSE :::bla where true or false TraverSE test:::xxx where false TraverSE :::ttt where true END END END where 1 = 1`
	expectedOutput = `
get
  value: "bla"
  traverse
    value: ":::bla"
    where
      or
        true
        false
    traverse
      value: "test:::xxx"
      where
        false
      traverse
        value: ":::ttt"
        where
          true
  where
    =
      value: "1"
      value: "1"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	// Test functions

	input = `
GeT Song where @a() or @count("File:File:StoredData:Data") > 1 and @boolfunc1(123,"test", aaa)`
	expectedOutput = `
get
  value: "Song"
  where
    or
      func
        value: "a"
      and
        >
          func
            value: "count"
            value: "File:File:"...
          value: "1"
        func
          value: "boolfunc1"
          value: "123"
          value: "test"
          value: "aaa"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}
}

func TestShowParsing(t *testing.T) {

	// Test simple show expression

	input := `
get song where true primary 1:song show name, state`
	expectedOutput := `
get
  value: "song"
  where
    true
  primary
    value: "1:song"
  show
    showterm: "name"
    showterm: "state"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	input = `
get song where true primary 1:song show name, state, @test(12, r"34") AS Bla FORMAT x, key`
	expectedOutput = `
get
  value: "song"
  where
    true
  primary
    value: "1:song"
  show
    showterm: "name"
    showterm: "state"
    showterm
      func
        value: "test"
        value: "12"
        value: "34"
      as
        value: "Bla"
      format
        value: "x"
    showterm: "key"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	input = `
get song where true primary 1:song show @test(12, r"34") format x`
	expectedOutput = `
get
  value: "song"
  where
    true
  primary
    value: "1:song"
  show
    showterm
      func
        value: "test"
        value: "12"
        value: "34"
      format
        value: "x"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	input = `
get song where true primary 1:song show 
Song:title AS r'Title (mytitle)',
r'Song!2:t title' AS "Title test" FORMAT text:bla_bla_blub:dudududu,
x:kind`
	expectedOutput = `
get
  value: "song"
  where
    true
  primary
    value: "1:song"
  show
    showterm: "Song:title"
      as
        value: "Title (myt"...
    showterm: "Song!2:t t"...
      as
        value: "Title test"
      format
        value: "text:bla_b"...
    showterm: "x:kind"
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	input = `
get song where true // 'div' show bla wIth orderinG(ASCending aa,Descending bb), FILTERING(ISNOTNULL test2,UNIQUE test3, uniquecount test3), nulltraversal(true)`
	expectedOutput = `
get
  value: "song"
  where
    divint
      true
      value: "div"
  show
    showterm: "bla"
  with
    ordering
      asc
        value: "aa"
      desc
        value: "bb"
    filtering
      isnotnull
        value: "test2"
      unique
        value: "test3"
      uniquecount
        value: "test3"
    nulltraversal
      true
`[1:]

	if res, err := Parse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}
}

func TestParserErrorCases(t *testing.T) {

	if res, err := ParseWithRuntime("mytest", "", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET r\"aa", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Lexical error (Invalid node kind 'r\"aa' - can only contain [a-zA-Z0-9_]) (Line:1 Pos:5)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "= GET", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Term cannot start an expression (=) (Line:1 Pos:1)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "get a where 1 (", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Term can only start an expression (() (Line:1 Pos:15)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "get a where (=", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Term cannot start an expression (=) (Line:1 Pos:14)" {
		t.Error("Unexpected result", res, err)
		return
	}

	// Test "Get" parsing with invalid lexer output

	res, err := testParserRun([]LexToken{
		LexToken{TokenGET, 1, "", 1, 1},
		LexToken{TokenGET, 1, "", 1, 1},
		LexToken{TokenEOF, 1, "", 1, 1},
	})
	if err.Error() != "Parse error in special test: Unexpected term (Line:1 Pos:1)" {
		t.Error("Unexpected result", res, err)
		return
	}

	res, err = testParserRun([]LexToken{
		LexToken{TokenLOOKUP, 1, "", 1, 1},
		LexToken{TokenGET, 1, "", 1, 1},
		LexToken{TokenEOF, 1, "", 1, 1},
	})
	if err.Error() != "Parse error in special test: Unexpected term (Line:1 Pos:1)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "lookup x", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "lookup x '123', GET", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term (GET) (Line:1 Pos:17)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "lookup x '123' GET", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x FROM GeT", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term (GeT) (Line:1 Pos:12)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x traverse GeT", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term (GeT) (Line:1 Pos:16)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x traverse ::: GeT", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x where @where(", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term (where) (Line:1 Pos:14)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x where @xxx)", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term ()) (Line:1 Pos:17)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x where @xxx(12", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end (Line:1 Pos:18)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x where @xxx(abc,", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x show a AS get", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term (get) (Line:1 Pos:17)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x show x, a FORMAT get", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term (get) (Line:1 Pos:24)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "GET x show @bla(1,", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "get a with ordering)", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected term ()) (Line:1 Pos:20)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "get a with ordering(=", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Term cannot start an expression (=) (Line:1 Pos:21)" {
		t.Error("Unexpected result", res, err)
		return
	}

	if res, err := ParseWithRuntime("mytest", "get a where [1,2", &TestRuntimeProvider{}); err.Error() !=
		"Parse error in mytest: Unexpected end" {
		t.Error("Unexpected result", res, err)
		return
	}

	var TokenUnknown LexTokenID = -5

	res, err = testParserRun([]LexToken{
		LexToken{TokenUnknown, 1, "", 1, 1},
		LexToken{TokenEOF, 1, "", 1, 1},
	})
	if err.Error() != "Parse error in special test: Unknown term (id:-5 (\"\")) (Line:1 Pos:1)" {
		t.Error("Unexpected result", res, err)
		return
	}

	res, err = testParserRun([]LexToken{
		LexToken{TokenVALUE, 1, "", 1, 1},
		LexToken{TokenMINUS, 1, "", 1, 1},
		LexToken{TokenUnknown, 1, "", 1, 1},
		LexToken{TokenEOF, 1, "", 1, 1},
	})
	if err.Error() != "Parse error in special test: Unknown term (id:-5 (\"\")) (Line:1 Pos:1)" {
		t.Error("Unexpected result", res, err)
		return
	}
}

func TestAstPlainRepresentation(t *testing.T) {

	input := `
get song where true // 'div' show bla wIth orderinG(ASCending aa,Descending bb), FILTERING(ISNOTNULL test2,UNIQUE test3, uniquecount test3), nulltraversal(true)`
	expectedOutput := `
get
  value: "song"
  where
    divint
      true
      value: "div"
  show
    showterm: "bla"
  with
    ordering
      asc
        value: "aa"
      desc
        value: "bb"
    filtering
      isnotnull
        value: "test2"
      unique
        value: "test3"
      uniquecount
        value: "test3"
    nulltraversal
      true
`[1:]

	res, err := Parse("mytest", input)

	if err != nil || fmt.Sprint(res) != expectedOutput {
		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
		return
	}

	plainres := res.Plain()

	jsonplainres, err := json.Marshal(plainres)
	if err != nil {
		t.Error(err)
		return
	}

	out := bytes.Buffer{}
	err = json.Indent(&out, []byte(jsonplainres), "", "  ")
	if err != nil {
		t.Error(err)
		return
	}

	if out.String() != `
{
  "children": [
    {
      "name": "value",
      "value": "song"
    },
    {
      "children": [
        {
          "children": [
            {
              "name": "true",
              "value": "true"
            },
            {
              "name": "value",
              "value": "div"
            }
          ],
          "name": "divint",
          "value": "//"
        }
      ],
      "name": "where",
      "value": "where"
    },
    {
      "children": [
        {
          "name": "showterm",
          "value": "bla"
        }
      ],
      "name": "show",
      "value": "show"
    },
    {
      "children": [
        {
          "children": [
            {
              "children": [
                {
                  "name": "value",
                  "value": "aa"
                }
              ],
              "name": "asc",
              "value": "ASCending"
            },
            {
              "children": [
                {
                  "name": "value",
                  "value": "bb"
                }
              ],
              "name": "desc",
              "value": "Descending"
            }
          ],
          "name": "ordering",
          "value": "orderinG"
        },
        {
          "children": [
            {
              "children": [
                {
                  "name": "value",
                  "value": "test2"
                }
              ],
              "name": "isnotnull",
              "value": "ISNOTNULL"
            },
            {
              "children": [
                {
                  "name": "value",
                  "value": "test3"
                }
              ],
              "name": "unique",
              "value": "UNIQUE"
            },
            {
              "children": [
                {
                  "name": "value",
                  "value": "test3"
                }
              ],
              "name": "uniquecount",
              "value": "uniquecount"
            }
          ],
          "name": "filtering",
          "value": "FILTERING"
        },
        {
          "children": [
            {
              "name": "true",
              "value": "true"
            }
          ],
          "name": "nulltraversal",
          "value": "nulltraversal"
        }
      ],
      "name": "with",
      "value": "wIth"
    }
  ],
  "name": "get",
  "value": "get"
}`[1:] {
		t.Error("Unexpected result: ", out.String())
		return
	}

	// Now convert the plain ast back into a normal AST and pretty print the result

	astfromplain, err := ASTFromPlain(plainres)
	if err != nil {
		t.Error(err)
		return
	}

	//  Check that the generated AST is equal to the expected output

	if fmt.Sprint(astfromplain) != expectedOutput {
		t.Error("Unexpected output:", astfromplain)
		return
	}

	ppquery, err := PrettyPrint(astfromplain)
	if err != nil {
		t.Error(err)
		return
	}

	if ppquery != `
get song where true // div
show
  bla 
with
  ordering(ascending aa, descending bb),
  filtering(isnotnull test2, unique test3, uniquecount test3),
  nulltraversal(true)`[1:] {
		t.Error("Unexpected output:", ppquery)
		return
	}

	// Test parsing from JSON (this will produce a []interface{} for children)

	data := make(map[string]interface{})
	json.NewDecoder(bytes.NewBufferString(`{
		"name"     : "get",
		"value"    : "get",
		"children" : [{
			"name"     : "value",
			"value"    : "bla"
		}]
	}`)).Decode(&data)

	astfromplain, err = ASTFromPlain(data)
	if err != nil {
		t.Error(err)
		return
	}

	if fmt.Sprint(astfromplain) != `
get
  value: "bla"
`[1:] {
		t.Error("Unexpected result:", astfromplain)
		return
	}

	// Test error message

	if _, err := ASTFromPlain(map[string]interface{}{
		"name": "bla",
	}); err.Error() != "Found plain ast node without a value: map[name:bla]" {
		t.Error("Unexpected error:", err)
		return
	}

	if _, err := ASTFromPlain(map[string]interface{}{
		"name":  "bla",
		"value": "",
		"children": []map[string]interface{}{map[string]interface{}{
			"fame": "bla",
		}},
	}); err.Error() != "Found plain ast node without a name: map[fame:bla]" {
		t.Error("Unexpected error:", err)
		return
	}
}

/*
Special function to test lexer runs which might be prevented by the actual lexer.
*/
func testParserRun(tokens []LexToken) (*ASTNode, error) {

	// Create channel which is filled with the given lex tokens

	tokenChan := make(chan LexToken)
	run := func() {
		for _, item := range tokens {
			tokenChan <- item
		}
	}
	go run()

	// Create parser which processes the given tokens

	p := &parser{"special test", nil, tokenChan, nil}

	node, err := p.next()

	if err != nil {
		return nil, err
	}

	p.node = node

	return p.run(0)
}