Browse Source

feat: Adding function declaration parsing to ECAL

Matthias Ladkau 3 years ago
parent
commit
89bf6e0d31

+ 0 - 3
lang/ecal/README.md

@@ -42,9 +42,6 @@ suppresses | A list of sink names which should be suppressed if this sink is exe
 Example:
 ```
 sink "mysink"
-    r"
-    A comment describing the sink.
-    "
     kindmatch [ foo.bar.* ],
     scopematch [ "data.read", "data.write" ],
     statematch { a : 1, b : NULL },

+ 17 - 4
lang/ecal/parser/const.go

@@ -64,9 +64,12 @@ const (
 
 	TokenPRECOMMENT  // Comment /* ... */
 	TokenPOSTCOMMENT // Comment # ...
-	TokenSTRING      // String constant
-	TokenNUMBER      // Number constant
-	TokenIDENTIFIER  // Idendifier
+
+	// Value tokens
+
+	TokenSTRING     // String constant
+	TokenNUMBER     // Number constant
+	TokenIDENTIFIER // Idendifier
 
 	// Constructed tokens which are generated by the parser not the lexer
 
@@ -75,6 +78,7 @@ const (
 	TokenCOMPACCESS // Access to a composition structure
 	TokenLIST       // List value
 	TokenMAP        // MAP value
+	TokenPARAMS     // Function parameters
 	TokenGUARD      // Guard expressions for conditional statements
 
 	TOKENodeSYMBOLS // Used to separate symbols from other tokens in this list
@@ -106,6 +110,7 @@ const (
 	// Grouping
 
 	TokenCOLON
+	TokenEQUAL
 
 	// Arithmetic operators
 
@@ -139,6 +144,7 @@ const (
 	// Function definition
 
 	TokenFUNC
+	TokenRETURN
 
 	// Boolean operators
 
@@ -199,6 +205,7 @@ const (
 	NodeCOMPACCESS = "compaccess" // Composition structure access
 	NodeLIST       = "list"       // List value
 	NodeMAP        = "map"        // Map value
+	NodePARAMS     = "params"     // Function parameters
 
 	// Condition operators
 
@@ -211,7 +218,8 @@ const (
 
 	// Separators
 
-	NodeKVP = "kvp" // Key-value pair
+	NodeKVP    = "kvp"    // Key-value pair
+	NodePRESET = "preset" // Preset value
 
 	// Arithmetic operators
 
@@ -230,6 +238,11 @@ const (
 
 	NodeIMPORT = "import"
 
+	// Function definition
+
+	NodeFUNC   = "function"
+	NodeRETURN = "return"
+
 	// Boolean operators
 
 	NodeAND = "and"

+ 8 - 3
lang/ecal/parser/lexer.go

@@ -190,7 +190,8 @@ var KeywordMap = map[string]LexTokenID{
 
 	// Function definition
 
-	"func": TokenFUNC,
+	"func":   TokenFUNC,
+	"return": TokenRETURN,
 
 	// Boolean operators
 
@@ -252,13 +253,17 @@ var SymbolMap = map[string]LexTokenID{
 	"{": TokenLBRACE,
 	"}": TokenRBRACE,
 
-	// Sequence symbols
+	// Separators
 
 	".": TokenDOT,
 	",": TokenCOMMA,
-	":": TokenCOLON,
 	";": TokenSEMICOLON,
 
+	// Grouping
+
+	":": TokenCOLON,
+	"=": TokenEQUAL,
+
 	// Arithmetic operators
 
 	"+":  TokenPLUS,

+ 2 - 2
lang/ecal/parser/lexer_test.go

@@ -69,14 +69,14 @@ func TestEquals(t *testing.T) {
 		return
 	}
 
-	if ok, msg := l[0].Equals(l[1], false); ok || msg != `ID is different 50 vs 7
+	if ok, msg := l[0].Equals(l[1], false); ok || msg != `ID is different 53 vs 7
 Pos is different 0 vs 5
 Val is different not vs test
 Identifier is different false vs true
 Lline is different 1 vs 2
 Lpos is different 1 vs 2
 {
-  "ID": 50,
+  "ID": 53,
   "Pos": 0,
   "Val": "not",
   "Identifier": false,

+ 153 - 1
lang/ecal/parser/parser.go

@@ -36,6 +36,7 @@ func init() {
 		TokenCOMPACCESS: {NodeCOMPACCESS, nil, nil, nil, nil, 0, nil, nil},
 		TokenLIST:       {NodeLIST, nil, nil, nil, nil, 0, nil, nil},
 		TokenMAP:        {NodeMAP, nil, nil, nil, nil, 0, nil, nil},
+		TokenPARAMS:     {NodePARAMS, nil, nil, nil, nil, 0, nil, nil},
 		/*
 			TokenGUARD:      {NodeGUARD, nil, nil, nil, 0, nil, nil},
 		*/
@@ -67,6 +68,7 @@ func init() {
 		// Grouping
 
 		TokenCOLON: {NodeKVP, nil, nil, nil, nil, 60, nil, ldInfix},
+		TokenEQUAL: {NodePRESET, nil, nil, nil, nil, 60, nil, ldInfix},
 
 		// Arithmetic operators
 
@@ -86,10 +88,28 @@ func init() {
 		TokenIMPORT: {NodeIMPORT, nil, nil, nil, nil, 0, ndImport, nil},
 		TokenAS:     {"", nil, nil, nil, nil, 0, ndImport, nil},
 
+		/*
+			// Sink definition
+
+			TokenSINK
+			TokenKINDMATCH
+			TokenSCOPEMATCH
+			TokenSTATEMATCH
+			TokenPRIORITY
+			TokenSUPPRESSES
+
+
+		*/
+
+		// Function definition
+
+		TokenFUNC:   {NodeFUNC, nil, nil, nil, nil, 0, ndFunc, nil},
+		TokenRETURN: {NodeRETURN, nil, nil, nil, nil, 0, ndReturn, nil},
+
 		// Boolean operators
 
-		TokenOR:  {NodeOR, nil, nil, nil, nil, 30, nil, ldInfix},
 		TokenAND: {NodeAND, nil, nil, nil, nil, 40, nil, ldInfix},
+		TokenOR:  {NodeOR, nil, nil, nil, nil, 30, nil, ldInfix},
 		TokenNOT: {NodeNOT, nil, nil, nil, nil, 20, ndPrefix, nil},
 
 		// Condition operators
@@ -105,6 +125,20 @@ func init() {
 		TokenFALSE: {NodeFALSE, nil, nil, nil, nil, 0, ndTerm, nil},
 		TokenTRUE:  {NodeTRUE, nil, nil, nil, nil, 0, ndTerm, nil},
 		TokenNULL:  {NodeNULL, nil, nil, nil, nil, 0, ndTerm, nil},
+
+		/*
+			// Conditional statements
+
+			TokenIF
+			TokenELIF
+			TokenELSE
+
+			// Loop statements
+
+			TokenFOR
+			TokenBREAK
+			TokenCONTINUE
+		*/
 	}
 }
 
@@ -370,6 +404,73 @@ func ndImport(p *parser, self *ASTNode) (*ASTNode, error) {
 	return self, err
 }
 
+/*
+ndFunc is used to parse function definitions.
+*/
+func ndFunc(p *parser, self *ASTNode) (*ASTNode, error) {
+
+	// Must specify a function name
+
+	err := acceptChild(p, self, TokenIDENTIFIER)
+
+	// Read in parameters
+
+	if err == nil {
+		err = skipToken(p, TokenLPAREN)
+
+		params := astNodeMap[TokenPARAMS].instance(p, nil)
+		self.Children = append(self.Children, params)
+
+		for err == nil && p.node.Token.ID != TokenRPAREN {
+
+			// Parse all the expressions inside
+
+			exp, err := p.run(0)
+			if err == nil {
+				params.Children = append(params.Children, exp)
+
+				if p.node.Token.ID == TokenCOMMA {
+					err = skipToken(p, TokenCOMMA)
+				}
+			}
+		}
+
+		if err == nil {
+			err = skipToken(p, TokenRPAREN)
+		}
+	}
+
+	if err == nil {
+
+		// Parse the body
+
+		self, err = parseInnerStatements(p, self)
+	}
+
+	return self, err
+}
+
+/*
+ndReturn is used to parse return statements.
+*/
+func ndReturn(p *parser, self *ASTNode) (*ASTNode, error) {
+	var err error
+
+	if self.Token.Lline == p.node.Token.Lline {
+		var val *ASTNode
+
+		// Consume the next expression only if it is on the same line
+
+		val, err = p.run(0)
+
+		if err == nil {
+			self.Children = append(self.Children, val)
+		}
+	}
+
+	return self, err
+}
+
 /*
 ndIdentifier is to parse identifiers and function calls.
 */
@@ -612,3 +713,54 @@ func acceptChild(p *parser, self *ASTNode, id LexTokenID) error {
 
 	return p.newParserError(ErrUnexpectedToken, current.Token.Val, *current.Token)
 }
+
+/*
+parseInnerStatements collects the inner statements of a block statement. It
+is assumed that a block statement starts with a left brace '{' and ends with
+a right brace '}'.
+*/
+func parseInnerStatements(p *parser, self *ASTNode) (*ASTNode, error) {
+
+	// Must start with an opening brace
+
+	if err := skipToken(p, TokenLBRACE); err != nil {
+		return nil, err
+	}
+
+	// Always create a statements node
+
+	st := astNodeMap[TokenSTATEMENTS].instance(p, nil)
+	self.Children = append(self.Children, st)
+
+	// Check if there are actually children
+
+	if p.node != nil && p.node.Token.ID != TokenRBRACE {
+
+		n, err := p.run(0)
+
+		if p.node != nil && p.node.Token.ID != TokenEOF {
+
+			st.Children = append(st.Children, n)
+
+			for hasMoreStatements(p, n) {
+
+				if p.node.Token.ID == TokenSEMICOLON {
+					skipToken(p, TokenSEMICOLON)
+				} else if p.node.Token.ID == TokenRBRACE {
+					break
+				}
+
+				n, err = p.run(0)
+				st.Children = append(st.Children, n)
+			}
+		}
+
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	// Must end with a closing brace
+
+	return self, skipToken(p, TokenRBRACE)
+}

+ 2 - 2
lang/ecal/parser/parser_exp_test.go

@@ -295,9 +295,9 @@ statements
 		return
 	}
 
-	input = `print(123); x := { z : "foo", y : "bar", z : "zzz" }; foo = y == 1`
+	input = `print(123); x := { z : "foo", y : "bar", z : "zzz" }; foo £ y == 1`
 	if _, err := UnitTestParse("mytest", input); err.Error() !=
-		"Parse error in mytest: Lexical error (Cannot parse identifier '='. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character) (Line:1 Pos:59)" {
+		"Parse error in mytest: Lexical error (Cannot parse identifier '£'. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character) (Line:1 Pos:59)" {
 		t.Error(err)
 		return
 	}

+ 68 - 0
lang/ecal/parser/parser_func_test.go

@@ -34,6 +34,74 @@ statements
 	}
 }
 
+func TestFuncParsing(t *testing.T) {
+
+	input := `import "foo/bar.ecal" as foobar
+
+func myfunc(a, b, c=1) {
+  foo := a and b and c
+  return foo
+}
+`
+	expectedOutput := `
+statements
+  import
+    string: 'foo/bar.ecal'
+    identifier: foobar
+  function
+    identifier: myfunc
+    params
+      identifier: a
+      identifier: b
+      preset
+        identifier: c
+        number: 1
+    statements
+      :=
+        identifier: foo
+        and
+          and
+            identifier: a
+            identifier: b
+          identifier: c
+      return
+        identifier: foo
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+
+	input = `
+func myfunc() {
+  a := 1
+  return
+  b := 2
+  return
+}
+`
+	expectedOutput = `
+function
+  identifier: myfunc
+  params
+  statements
+    :=
+      identifier: a
+      number: 1
+    return
+    :=
+      identifier: b
+      number: 2
+    return
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
+}
+
 func TestFunctionCalling(t *testing.T) {
 
 	input := `import "foo/bar.ecal" as foobar

+ 23 - 1
lang/ecal/parser/prettyprinter.go

@@ -41,6 +41,9 @@ func init() {
 		// NodeSTATEMENTS - Special case (handled in code)
 		// NodeFUNCCALL - Special case (handled in code)
 		NodeCOMPACCESS + "_1": template.Must(template.New(NodeCOMPACCESS).Parse("[{{.c1}}]")),
+		// TokenLIST - Special case (handled in code)
+		// TokenMAP - Special case (handled in code)
+		// TokenPARAMS - Special case (handled in code)
 
 		/*
 
@@ -71,6 +74,12 @@ func init() {
 		NodeMODINT + "_2": template.Must(template.New(NodeMODINT).Parse("{{.c1}} % {{.c2}}")),
 		NodeDIVINT + "_2": template.Must(template.New(NodeDIVINT).Parse("{{.c1}} // {{.c2}}")),
 
+		// Function definition
+
+		NodeFUNC + "_3":   template.Must(template.New(NodeDIVINT).Parse("func {{.c1}}{{.c2}} {\n{{.c3}}}")),
+		NodeRETURN:        template.Must(template.New(NodeDIVINT).Parse("return")),
+		NodeRETURN + "_1": template.Must(template.New(NodeDIVINT).Parse("return {{.c1}}")),
+
 		// Boolean operators
 
 		NodeOR + "_2":  template.Must(template.New(NodeGEQ).Parse("{{.c1}} or {{.c2}}")),
@@ -94,7 +103,8 @@ func init() {
 
 		// Separators
 
-		NodeKVP + "_2": template.Must(template.New(NodeLT).Parse("{{.c1}} : {{.c2}}")),
+		NodeKVP + "_2":    template.Must(template.New(NodeLT).Parse("{{.c1}} : {{.c2}}")),
+		NodePRESET + "_2": template.Must(template.New(NodeLT).Parse("{{.c1}}={{.c2}}")),
 
 		// Constants
 
@@ -234,6 +244,18 @@ func PrettyPrint(ast *ASTNode) (string, error) {
 			buf.WriteString(tempParam[fmt.Sprint("c", i)])
 			buf.WriteString("}")
 
+			return ppMetaData(ast, buf.String()), nil
+		} else if ast.Name == NodePARAMS {
+
+			buf.WriteString("(")
+			i := 1
+			for ; i < numChildren; i++ {
+				buf.WriteString(tempParam[fmt.Sprint("c", i)])
+				buf.WriteString(", ")
+			}
+			buf.WriteString(tempParam[fmt.Sprint("c", i)])
+			buf.WriteString(")")
+
 			return ppMetaData(ast, buf.String()), nil
 		}