feat: Adding map and list parsing to ECAL

lang/ecal/parser/const.go

@@ -101,9 +101,12 @@ const (
 
 	TokenDOT
 	TokenCOMMA
-	TokenCOLON
 	TokenSEMICOLON
 
+	// Grouping
+
+	TokenCOLON
+
 	// Arithmetic operators
 
 	TokenPLUS
@@ -117,8 +120,6 @@ const (
 
 	TokenASSIGN
 
-	// The colon '' has a context specific meaning and is checked by the parser
-
 	TOKENodeKEYWORDS // Used to separate keywords from other tokens in this list
 
 	// Import statement
@@ -194,8 +195,10 @@ const (
 	// Constructed tokens
 
 	NodeSTATEMENTS = "statements" // List of statements
-	NodeFUNCCALL   = "funccall"
-	NodeCOMPACCESS = "compaccess"
+	NodeFUNCCALL   = "funccall"   // Function call
+	NodeCOMPACCESS = "compaccess" // Composition structure access
+	NodeLIST       = "list"       // List value
+	NodeMAP        = "map"        // Map value
 
 	// Condition operators
 
@@ -206,6 +209,10 @@ const (
 	NodeGT  = ">"
 	NodeLT  = "<"
 
+	// Separators
+
+	NodeKVP = "kvp" // Key-value pair
+
 	// Arithmetic operators
 
 	NodePLUS   = "plus"

lang/ecal/parser/parser.go

@@ -34,9 +34,9 @@ func init() {
 		TokenSTATEMENTS: {NodeSTATEMENTS, nil, nil, nil, nil, 0, nil, nil},
 		TokenFUNCCALL:   {NodeFUNCCALL, nil, nil, nil, nil, 0, nil, nil},
 		TokenCOMPACCESS: {NodeCOMPACCESS, nil, nil, nil, nil, 0, nil, nil},
+		TokenLIST:       {NodeLIST, nil, nil, nil, nil, 0, nil, nil},
+		TokenMAP:        {NodeMAP, nil, nil, nil, nil, 0, nil, nil},
 		/*
-			TokenLIST:       {NodeLIST, nil, nil, nil, 0, nil, nil},
-			TokenMAP:        {NodeMAP, nil, nil, nil, 0, nil, nil},
 			TokenGUARD:      {NodeGUARD, nil, nil, nil, 0, nil, nil},
 		*/
 
@@ -51,10 +51,12 @@ func init() {
 
 		// Grouping symbols
 
-		TokenLBRACK: {"", nil, nil, nil, nil, 150, ndInner, nil},
-		TokenRBRACK: {"", nil, nil, nil, nil, 0, nil, nil},
 		TokenLPAREN: {"", nil, nil, nil, nil, 150, ndInner, nil},
 		TokenRPAREN: {"", nil, nil, nil, nil, 0, nil, nil},
+		TokenLBRACK: {"", nil, nil, nil, nil, 150, ndList, nil},
+		TokenRBRACK: {"", nil, nil, nil, nil, 0, nil, nil},
+		TokenLBRACE: {"", nil, nil, nil, nil, 150, ndMap, nil},
+		TokenRBRACE: {"", nil, nil, nil, nil, 0, nil, nil},
 
 		// Separators
 
@@ -62,6 +64,10 @@ func init() {
 		TokenCOMMA:     {"", nil, nil, nil, nil, 0, nil, nil},
 		TokenSEMICOLON: {"", nil, nil, nil, nil, 0, nil, nil},
 
+		// Grouping
+
+		TokenCOLON: {NodeKVP, nil, nil, nil, nil, 60, nil, ldInfix},
+
 		// Arithmetic operators
 
 		TokenPLUS:   {NodePLUS, nil, nil, nil, nil, 110, ndPrefix, ldInfix},
@@ -453,6 +459,72 @@ func ndIdentifier(p *parser, self *ASTNode) (*ASTNode, error) {
 	return self, parseMore(self)
 }
 
+/*
+ndList is used to collect elements of a list.
+*/
+func ndList(p *parser, self *ASTNode) (*ASTNode, error) {
+
+	// Create a list token
+
+	st := astNodeMap[TokenLIST].instance(p, self.Token)
+
+	// Get the inner expression
+
+	for p.node.Token.ID != TokenRBRACK {
+
+		// Parse all the expressions inside
+
+		exp, err := p.run(0)
+		if err != nil {
+			return nil, err
+		}
+
+		st.Children = append(st.Children, exp)
+
+		if p.node.Token.ID == TokenCOMMA {
+			skipToken(p, TokenCOMMA)
+		}
+	}
+
+	// Must have a closing bracket
+
+	return st, skipToken(p, TokenRBRACK)
+}
+
+/*
+ndMap is used to collect elements of a map.
+*/
+func ndMap(p *parser, self *ASTNode) (*ASTNode, error) {
+
+	// Create a map token
+
+	st := astNodeMap[TokenMAP].instance(p, self.Token)
+
+	// Get the inner expression
+
+	for p.node.Token.ID != TokenRBRACE {
+
+		// Parse all the expressions inside
+
+		exp, err := p.run(0)
+		if err != nil {
+			return nil, err
+		}
+
+		st.Children = append(st.Children, exp)
+
+		if p.node.Token.ID == TokenCOMMA {
+			if err := skipToken(p, TokenCOMMA); err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	// Must have a closing brace
+
+	return st, skipToken(p, TokenRBRACE)
+}
+
 // Standard left denotation functions
 // ==================================
 

lang/ecal/parser/parser_exp_test.go

@@ -0,0 +1,345 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package parser
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestSimpleExpressionParsing(t *testing.T) {
+
+	// Test error output
+
+	input := `"bl\*a"conversion`
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Lexical error (invalid syntax while parsing string) (Line:1 Pos:1)" {
+		t.Error(err)
+		return
+	}
+
+	// Test incomplete expression
+
+	input = `a *`
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Unexpected end" {
+		t.Error(err)
+		return
+	}
+
+	input = `not ==`
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Term cannot start an expression (==) (Line:1 Pos:5)" {
+		t.Error(err)
+		return
+	}
+
+	input = `(==)`
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Term cannot start an expression (==) (Line:1 Pos:2)" {
+		t.Error(err)
+		return
+	}
+
+	input = "5 ( 5"
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Term can only start an expression (() (Line:1 Pos:3)" {
+		t.Error(err)
+		return
+	}
+
+	input = "5 + \""
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Lexical error (Unexpected end while reading string value (unclosed quotes)) (Line:1 Pos:5)" {
+		t.Error(err)
+		return
+	}
+
+	// Test prefix operator
+
+	input = ` + a - -5`
+	expectedOutput := `
+minus
+  plus
+    identifier: a
+  minus
+    number: 5
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+}
+
+func TestArithmeticParsing(t *testing.T) {
+	input := "a + b * 5 /2"
+	expectedOutput := `
+plus
+  identifier: a
+  div
+    times
+      identifier: b
+      number: 5
+    number: 2
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	// Test brackets
+
+	input = "a + 1 * (5 + 6)"
+	expectedOutput = `
+plus
+  identifier: a
+  times
+    number: 1
+    plus
+      number: 5
+      number: 6
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	// Test needless brackets
+
+	input = "(a + 1) * (5 / (6 - 2))"
+	expectedOutput = `
+times
+  plus
+    identifier: a
+    number: 1
+  div
+    number: 5
+    minus
+      number: 6
+      number: 2
+`[1:]
+
+	// Pretty printer should get rid of the needless brackets
+
+	res, err := UnitTestParseWithPPResult("mytest", input, "(a + 1) * 5 / (6 - 2)")
+	if err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+}
+
+func TestLogicParsing(t *testing.T) {
+	input := "not (a + 1) * 5 and tRue == false or not 1 - 5 != test"
+	expectedOutput := `
+or
+  and
+    not
+      times
+        plus
+          identifier: a
+          number: 1
+        number: 5
+    ==
+      true
+      false
+  not
+    !=
+      minus
+        number: 1
+        number: 5
+      identifier: test
+`[1:]
+
+	res, err := UnitTestParseWithPPResult("mytest", input, "not (a + 1) * 5 and true == false or not 1 - 5 != test")
+
+	if err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	input = "a > b or a <= p or b hasSuffix 'test' or c hasPrefix 'test' and x < 4 or x >= 10"
+	expectedOutput = `
+or
+  or
+    or
+      or
+        >
+          identifier: a
+          identifier: b
+        <=
+          identifier: a
+          identifier: p
+      hassuffix
+        identifier: b
+        string: 'test'
+    and
+      hasprefix
+        identifier: c
+        string: 'test'
+      <
+        identifier: x
+        number: 4
+  >=
+    identifier: x
+    number: 10
+`[1:]
+
+	res, err = UnitTestParseWithPPResult("mytest", input, `a > b or a <= p or b hassuffix "test" or c hasprefix "test" and x < 4 or x >= 10`)
+
+	if err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	input = "(a in null or c notin d) and false like 9 or x // 6 > 2 % 1"
+	expectedOutput = `
+or
+  and
+    or
+      in
+        identifier: a
+        null
+      notin
+        identifier: c
+        identifier: d
+    like
+      false
+      number: 9
+  >
+    divint
+      identifier: x
+      number: 6
+    modint
+      number: 2
+      number: 1
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+}
+
+func TestCompositionStructureParsing(t *testing.T) {
+
+	// Assignment of map
+
+	input := `x := { z : "foo", y : "bar", z : "zzz" }`
+	expectedOutput := `
+:=
+  identifier: x
+  map
+    kvp
+      identifier: z
+      string: 'foo'
+    kvp
+      identifier: y
+      string: 'bar'
+    kvp
+      identifier: z
+      string: 'zzz'
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	input = `x := { ==`
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Term cannot start an expression (==) (Line:1 Pos:8)" {
+		t.Error(err)
+		return
+	}
+
+	// Statement separator
+
+	input = `print(123); x := { z : "foo", y : "bar", z : "zzz" }; foo := y == 1`
+	expectedOutput = `
+statements
+  identifier: print
+    funccall
+      number: 123
+  :=
+    identifier: x
+    map
+      kvp
+        identifier: z
+        string: 'foo'
+      kvp
+        identifier: y
+        string: 'bar'
+      kvp
+        identifier: z
+        string: 'zzz'
+  :=
+    identifier: foo
+    ==
+      identifier: y
+      number: 1
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	input = `print(123); x := { z : "foo", y : "bar", z : "zzz" }; foo = y == 1`
+	if _, err := UnitTestParse("mytest", input); err.Error() !=
+		"Parse error in mytest: Lexical error (Cannot parse identifier '='. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character) (Line:1 Pos:59)" {
+		t.Error(err)
+		return
+	}
+
+	input = `x := [1,2]
+[a,b] := x`
+	expectedOutput = `
+statements
+  :=
+    identifier: x
+    list
+      number: 1
+      number: 2
+  :=
+    list
+      identifier: a
+      identifier: b
+    identifier: x
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	input = `x := [1,2];[a,b] := x`
+	expectedOutput = `
+statements
+  :=
+    identifier: x
+    list
+      number: 1
+      number: 2
+  :=
+    list
+      identifier: a
+      identifier: b
+    identifier: x
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+}

lang/ecal/parser/parser_main_test.go

@@ -125,220 +125,3 @@ number: 1 #  foo   foo bar
 		return
 	}
 }
-
-func TestSimpleExpressionParsing(t *testing.T) {
-
-	// Test error output
-
-	input := `"bl\*a"conversion`
-	if _, err := UnitTestParse("mytest", input); err.Error() !=
-		"Parse error in mytest: Lexical error (invalid syntax while parsing string) (Line:1 Pos:1)" {
-		t.Error(err)
-		return
-	}
-
-	// Test incomplete expression
-
-	input = `a *`
-	if _, err := UnitTestParse("mytest", input); err.Error() !=
-		"Parse error in mytest: Unexpected end" {
-		t.Error(err)
-		return
-	}
-
-	input = `not ==`
-	if _, err := UnitTestParse("mytest", input); err.Error() !=
-		"Parse error in mytest: Term cannot start an expression (==) (Line:1 Pos:5)" {
-		t.Error(err)
-		return
-	}
-
-	input = `(==)`
-	if _, err := UnitTestParse("mytest", input); err.Error() !=
-		"Parse error in mytest: Term cannot start an expression (==) (Line:1 Pos:2)" {
-		t.Error(err)
-		return
-	}
-
-	input = "5 ( 5"
-	if _, err := UnitTestParse("mytest", input); err.Error() !=
-		"Parse error in mytest: Term can only start an expression (() (Line:1 Pos:3)" {
-		t.Error(err)
-		return
-	}
-
-	input = "5 + \""
-	if _, err := UnitTestParse("mytest", input); err.Error() !=
-		"Parse error in mytest: Lexical error (Unexpected end while reading string value (unclosed quotes)) (Line:1 Pos:5)" {
-		t.Error(err)
-		return
-	}
-
-	// Test prefix operator
-
-	input = ` + a - -5`
-	expectedOutput := `
-minus
-  plus
-    identifier: a
-  minus
-    number: 5
-`[1:]
-
-	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
-		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
-		return
-	}
-
-}
-
-func TestArithmeticParsing(t *testing.T) {
-	input := "a + b * 5 /2"
-	expectedOutput := `
-plus
-  identifier: a
-  div
-    times
-      identifier: b
-      number: 5
-    number: 2
-`[1:]
-
-	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
-		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
-		return
-	}
-
-	// Test brackets
-
-	input = "a + 1 * (5 + 6)"
-	expectedOutput = `
-plus
-  identifier: a
-  times
-    number: 1
-    plus
-      number: 5
-      number: 6
-`[1:]
-
-	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
-		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
-		return
-	}
-
-	// Test needless brackets
-
-	input = "(a + 1) * (5 / (6 - 2))"
-	expectedOutput = `
-times
-  plus
-    identifier: a
-    number: 1
-  div
-    number: 5
-    minus
-      number: 6
-      number: 2
-`[1:]
-
-	// Pretty printer should get rid of the needless brackets
-
-	res, err := UnitTestParseWithPPResult("mytest", input, "(a + 1) * 5 / (6 - 2)")
-	if err != nil || fmt.Sprint(res) != expectedOutput {
-		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
-		return
-	}
-}
-
-func TestLogicParsing(t *testing.T) {
-	input := "not (a + 1) * 5 and tRue == false or not 1 - 5 != test"
-	expectedOutput := `
-or
-  and
-    not
-      times
-        plus
-          identifier: a
-          number: 1
-        number: 5
-    ==
-      true
-      false
-  not
-    !=
-      minus
-        number: 1
-        number: 5
-      identifier: test
-`[1:]
-
-	res, err := UnitTestParseWithPPResult("mytest", input, "not (a + 1) * 5 and true == false or not 1 - 5 != test")
-
-	if err != nil || fmt.Sprint(res) != expectedOutput {
-		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
-		return
-	}
-
-	input = "a > b or a <= p or b hasSuffix 'test' or c hasPrefix 'test' and x < 4 or x >= 10"
-	expectedOutput = `
-or
-  or
-    or
-      or
-        >
-          identifier: a
-          identifier: b
-        <=
-          identifier: a
-          identifier: p
-      hassuffix
-        identifier: b
-        string: 'test'
-    and
-      hasprefix
-        identifier: c
-        string: 'test'
-      <
-        identifier: x
-        number: 4
-  >=
-    identifier: x
-    number: 10
-`[1:]
-
-	res, err = UnitTestParseWithPPResult("mytest", input, `a > b or a <= p or b hassuffix "test" or c hasprefix "test" and x < 4 or x >= 10`)
-
-	if err != nil || fmt.Sprint(res) != expectedOutput {
-		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
-		return
-	}
-
-	input = "(a in null or c notin d) and false like 9 or x // 6 > 2 % 1"
-	expectedOutput = `
-or
-  and
-    or
-      in
-        identifier: a
-        null
-      notin
-        identifier: c
-        identifier: d
-    like
-      false
-      number: 9
-  >
-    divint
-      identifier: x
-      number: 6
-    modint
-      number: 2
-      number: 1
-`[1:]
-
-	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
-		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
-		return
-	}
-}

lang/ecal/parser/prettyprinter.go

@@ -92,6 +92,10 @@ func init() {
 		NodeGT + "_2":  template.Must(template.New(NodeGT).Parse("{{.c1}} > {{.c2}}")),
 		NodeLT + "_2":  template.Must(template.New(NodeLT).Parse("{{.c1}} < {{.c2}}")),
 
+		// Separators
+
+		NodeKVP + "_2": template.Must(template.New(NodeLT).Parse("{{.c1}} : {{.c2}}")),
+
 		// Constants
 
 		NodeTRUE:  template.Must(template.New(NodeTRUE).Parse("true")),
@@ -205,6 +209,31 @@ func PrettyPrint(ast *ASTNode) (string, error) {
 				}
 			}
 
+			return ppMetaData(ast, buf.String()), nil
+		} else if ast.Name == NodeLIST {
+
+			buf.WriteString("[")
+			i := 1
+			for ; i < numChildren; i++ {
+				buf.WriteString(tempParam[fmt.Sprint("c", i)])
+				buf.WriteString(", ")
+			}
+			buf.WriteString(tempParam[fmt.Sprint("c", i)])
+			buf.WriteString("]")
+
+			return ppMetaData(ast, buf.String()), nil
+
+		} else if ast.Name == NodeMAP {
+
+			buf.WriteString("{")
+			i := 1
+			for ; i < numChildren; i++ {
+				buf.WriteString(tempParam[fmt.Sprint("c", i)])
+				buf.WriteString(", ")
+			}
+			buf.WriteString(tempParam[fmt.Sprint("c", i)])
+			buf.WriteString("}")
+
 			return ppMetaData(ast, buf.String()), nil
 		}