Browse Source

feat: Adding serialization for ECAL ASTs

Matthias Ladkau 3 years ago
parent
commit
a192de5788

+ 10 - 1
lang/ecal/parser/const.go

@@ -51,7 +51,7 @@ Available lexer token types
 const (
 	TokenError LexTokenID = iota // Lexing error token with a message as val
 	TokenEOF                     // End-of-file token
-	TokenAny                     // Unspecified token (used when building an AST from a Go map structure)
+	TokenANY                     // Unspecified token (used when building an AST from a Go map structure)
 
 	TokenCOMMENT    // Comment
 	TokenSTRING     // String constant
@@ -157,8 +157,17 @@ const (
 	TokenFOR
 	TokenBREAK
 	TokenCONTINUE
+
+	TokenENDLIST
 )
 
+/*
+IsValidTokenID check if a given token ID is valid.
+*/
+func IsValidTokenID(value int) bool {
+	return value < int(TokenENDLIST)
+}
+
 /*
 Available parser AST node types
 */

+ 201 - 5
lang/ecal/parser/helper.go

@@ -12,6 +12,7 @@ package parser
 import (
 	"bytes"
 	"fmt"
+	"strconv"
 
 	"devt.de/krotik/common/datautil"
 	"devt.de/krotik/common/stringutil"
@@ -48,19 +49,73 @@ func (n *ASTNode) instance(p *parser, t *LexToken) *ASTNode {
 	return ret
 }
 
+/*
+Equal checks if this AST data equals another AST data. Returns also a message describing
+what is the found difference.
+*/
+func (n *ASTNode) Equals(other *ASTNode) (bool, string) {
+	return n.equalsPath(n.Name, other)
+}
+
+/*
+equalsPath checks if this AST data equals another AST data while preserving the search path.
+Returns also a message describing what is the found difference.
+*/
+func (n *ASTNode) equalsPath(path string, other *ASTNode) (bool, string) {
+	var res = true
+	var msg = ""
+
+	if n.Name != other.Name {
+		res = false
+		msg = fmt.Sprintf("Name is different %v vs %v\n", n.Name, other.Name)
+	}
+
+	if ok, tokenMSG := n.Token.Equals(*other.Token); !ok {
+		res = false
+		msg += fmt.Sprintf("Token is different:\n%v\n", tokenMSG)
+	}
+
+	if len(n.Children) != len(other.Children) {
+		res = false
+		msg = fmt.Sprintf("Number of children is different %v vs %v\n",
+			len(n.Children), len(other.Children))
+	} else {
+		for i, child := range n.Children {
+
+			// Check for different in children
+
+			if ok, childMSG := child.equalsPath(fmt.Sprintf("%v > %v", path, child.Name),
+				other.Children[i]); !ok {
+				return ok, childMSG
+			}
+		}
+	}
+
+	if msg != "" {
+		var buf bytes.Buffer
+		buf.WriteString("AST Nodes:\n")
+		n.levelString(0, &buf, 1)
+		buf.WriteString("vs\n")
+		other.levelString(0, &buf, 1)
+		msg = fmt.Sprintf("Path to difference: %v\n\n%v\n%v", path, msg, buf.String())
+	}
+
+	return res, msg
+}
+
 /*
 String returns a string representation of this token.
 */
 func (n *ASTNode) String() string {
 	var buf bytes.Buffer
-	n.levelString(0, &buf)
+	n.levelString(0, &buf, -1)
 	return buf.String()
 }
 
 /*
 levelString function to recursively print the tree.
 */
-func (n *ASTNode) levelString(indent int, buf *bytes.Buffer) {
+func (n *ASTNode) levelString(indent int, buf *bytes.Buffer, printChildren int) {
 
 	// Print current level
 
@@ -80,11 +135,152 @@ func (n *ASTNode) levelString(indent int, buf *bytes.Buffer) {
 
 	buf.WriteString("\n")
 
-	// Print children
+	if printChildren == -1 || printChildren > 0 {
+
+		if printChildren != -1 {
+			printChildren--
+		}
+
+		// Print children
+
+		for _, child := range n.Children {
+			child.levelString(indent+1, buf, printChildren)
+		}
+	}
+}
+
+/*
+ToJSON returns this ASTNode and all its children as a JSON object.
+*/
+func (n *ASTNode) ToJSONObject() map[string]interface{} {
+	ret := make(map[string]interface{})
+
+	ret["name"] = n.Name
+
+	lenChildren := len(n.Children)
+
+	if lenChildren > 0 {
+		children := make([]map[string]interface{}, lenChildren)
+		for i, child := range n.Children {
+			children[i] = child.ToJSONObject()
+		}
+
+		ret["children"] = children
+	}
+
+	// The value is what the lexer found in the source
+
+	if n.Token != nil {
+		ret["id"] = n.Token.ID
+		if n.Token.Val != "" {
+			ret["value"] = n.Token.Val
+		}
+		ret["identifier"] = n.Token.Identifier
+		ret["pos"] = n.Token.Pos
+		ret["line"] = n.Token.Lline
+		ret["linepos"] = n.Token.Lpos
+	}
+
+	return ret
+}
+
+/*
+ASTFromPlain creates an AST from a JSON Object.
+The following nested map structure is expected:
+
+	{
+		name     : <name of node>
+
+		// Optional node information
+		value    : <value of node>
+		children : [ <child nodes> ]
+
+		// Optional token information
+		id       : <token id>
+	}
+*/
+func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
+	var astChildren []*ASTNode
+	var nodeID LexTokenID = TokenANY
+	var pos, line, linepos int
+
+	name, ok := jsonAST["name"]
+	if !ok {
+		return nil, fmt.Errorf("Found json ast node without a name: %v", jsonAST)
+	}
+
+	if nodeIDString, ok := jsonAST["id"]; ok {
+		if nodeIDInt, err := strconv.Atoi(fmt.Sprint(nodeIDString)); err == nil && IsValidTokenID(nodeIDInt) {
+			nodeID = LexTokenID(nodeIDInt)
+		}
+	}
+
+	value, ok := jsonAST["value"]
+	if !ok {
+		value = ""
+	}
+
+	identifier, ok := jsonAST["identifier"]
+	if !ok {
+		identifier = false
+	}
+
+	if posString, ok := jsonAST["pos"]; ok {
+		pos, _ = strconv.Atoi(fmt.Sprint(posString))
+	} else {
+		pos = 0
+	}
+
+	if lineString, ok := jsonAST["line"]; ok {
+		line, _ = strconv.Atoi(fmt.Sprint(lineString))
+	} else {
+		line = 0
+	}
+
+	if lineposString, ok := jsonAST["linepos"]; ok {
+		linepos, _ = strconv.Atoi(fmt.Sprint(lineposString))
+	} else {
+		linepos = 0
+	}
+
+	// Create children
+
+	if children, ok := jsonAST["children"]; ok {
 
-	for _, child := range n.Children {
-		child.levelString(indent+1, buf)
+		if ic, ok := children.([]interface{}); ok {
+
+			// Do a list conversion if necessary - this is necessary when we parse
+			// JSON with map[string]interface{}
+
+			childrenList := make([]map[string]interface{}, len(ic))
+			for i := range ic {
+				childrenList[i] = ic[i].(map[string]interface{})
+			}
+
+			children = childrenList
+		}
+
+		for _, child := range children.([]map[string]interface{}) {
+
+			astChild, err := ASTFromJSONObject(child)
+			if err != nil {
+				return nil, err
+			}
+
+			astChildren = append(astChildren, astChild)
+		}
+	}
+
+	token := &LexToken{
+		nodeID,             // ID
+		pos,                // Pos
+		fmt.Sprint(value),  // Val
+		identifier == true, // Identifier
+		line,               // Lline
+		linepos,            // Lpos
 	}
+
+	return &ASTNode{fmt.Sprint(name), token, astChildren, nil, 0, nil, nil}, nil
 }
 
 // Look ahead buffer

+ 164 - 0
lang/ecal/parser/helper_test.go

@@ -13,6 +13,170 @@ import (
 	"testing"
 )
 
+func TestASTNode(t *testing.T) {
+
+	n, err := ParseWithRuntime("", "- 1", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	n2, err := ParseWithRuntime("", "-2", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	if ok, msg := n.Equals(n2); ok || msg != `Path to difference: minus > number
+
+Token is different:
+Pos is different 2 vs 1
+Val is different 1 vs 2
+Lpos is different 3 vs 2
+{
+  "ID": 5,
+  "Pos": 2,
+  "Val": "1",
+  "Identifier": false,
+  "Lline": 1,
+  "Lpos": 3
+}
+vs
+{
+  "ID": 5,
+  "Pos": 1,
+  "Val": "2",
+  "Identifier": false,
+  "Lline": 1,
+  "Lpos": 2
+}
+
+AST Nodes:
+number: 1
+vs
+number: 2
+` {
+		t.Error("Unexpected result: ", msg)
+		return
+	}
+
+	n, err = ParseWithRuntime("", "-1", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	n2, err = ParseWithRuntime("", "-a", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	if ok, msg := n.Equals(n2); ok || msg != `Path to difference: minus > number
+
+Name is different number vs identifier
+Token is different:
+ID is different 5 vs 6
+Val is different 1 vs a
+Identifier is different false vs true
+{
+  "ID": 5,
+  "Pos": 1,
+  "Val": "1",
+  "Identifier": false,
+  "Lline": 1,
+  "Lpos": 2
+}
+vs
+{
+  "ID": 6,
+  "Pos": 1,
+  "Val": "a",
+  "Identifier": true,
+  "Lline": 1,
+  "Lpos": 2
+}
+
+AST Nodes:
+number: 1
+vs
+identifier: a
+` {
+		t.Error("Unexpected result: ", msg)
+		return
+	}
+
+	n, err = ParseWithRuntime("", "- 1", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	n2, err = ParseWithRuntime("", "a - b", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	if ok, msg := n.Equals(n2); ok || msg != `Path to difference: minus
+
+Number of children is different 1 vs 2
+
+AST Nodes:
+minus
+  number: 1
+vs
+minus
+  identifier: a
+  identifier: b
+` {
+		t.Error("Unexpected result: ", msg)
+		return
+	}
+
+	// Test building an AST from an invalid
+
+	if _, err := ASTFromJSONObject(map[string]interface{}{
+		"value": "foo",
+	}); err == nil || err.Error() != "Found json ast node without a name: map[value:foo]" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	if _, err := ASTFromJSONObject(map[string]interface{}{
+		"name": "foo",
+		"children": []map[string]interface{}{
+			map[string]interface{}{
+				"value": "bar",
+			},
+		},
+	}); err == nil || err.Error() != "Found json ast node without a name: map[value:bar]" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	// Test population of missing information
+
+	if ast, err := ASTFromJSONObject(map[string]interface{}{
+		"name": "foo",
+	}); err != nil || ast.String() != "foo\n" || ast.Token.String() != `v:""` {
+		t.Error("Unexpected result: ", ast.Token.String(), ast.String(), err)
+		return
+	}
+
+	if ast, err := ASTFromJSONObject(map[string]interface{}{
+		"name": "foo",
+		"children": []map[string]interface{}{
+			map[string]interface{}{
+				"name": "bar",
+			},
+		},
+	}); err != nil || ast.String() != "foo\n  bar\n" || ast.Token.String() != `v:""` {
+		t.Error("Unexpected result: ", ast.Token.String(), ast.String(), err)
+		return
+	}
+}
+
 func TestLABuffer(t *testing.T) {
 
 	buf := NewLABuffer(Lex("test", "1 2 3 4 5 6 7 8 9"), 3)

+ 53 - 0
lang/ecal/parser/lexer.go

@@ -10,6 +10,8 @@
 package parser
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"regexp"
 	"strconv"
@@ -33,6 +35,57 @@ type LexToken struct {
 	Lpos       int        // Position in the input line this token appears
 }
 
+/*
+Equal checks if this LexToken equals another LexToken. Returns also a message describing
+what is the found difference.
+*/
+func (n LexToken) Equals(other LexToken) (bool, string) {
+	var res = true
+	var msg = ""
+
+	if n.ID != other.ID {
+		res = false
+		msg += fmt.Sprintf("ID is different %v vs %v\n", n.ID, other.ID)
+	}
+
+	if n.Pos != other.Pos {
+		res = false
+		msg += fmt.Sprintf("Pos is different %v vs %v\n", n.Pos, other.Pos)
+	}
+
+	if n.Val != other.Val {
+		res = false
+		msg += fmt.Sprintf("Val is different %v vs %v\n", n.Val, other.Val)
+	}
+
+	if n.Identifier != other.Identifier {
+		res = false
+		msg += fmt.Sprintf("Identifier is different %v vs %v\n", n.Identifier, other.Identifier)
+	}
+
+	if n.Lline != other.Lline {
+		res = false
+		msg += fmt.Sprintf("Lline is different %v vs %v\n", n.Lline, other.Lline)
+	}
+
+	if n.Lpos != other.Lpos {
+		res = false
+		msg += fmt.Sprintf("Lpos is different %v vs %v\n", n.Lpos, other.Lpos)
+	}
+
+	if msg != "" {
+		var buf bytes.Buffer
+		out, _ := json.MarshalIndent(n, "", "  ")
+		buf.WriteString(string(out))
+		buf.WriteString("\nvs\n")
+		out, _ = json.MarshalIndent(other, "", "  ")
+		buf.WriteString(string(out))
+		msg = fmt.Sprintf("%v%v", msg, buf.String())
+	}
+
+	return res, msg
+}
+
 /*
 PosString returns the position of this token in the origianl input as a string.
 */

+ 31 - 0
lang/ecal/parser/lexer_test.go

@@ -59,6 +59,37 @@ func TestNextItem(t *testing.T) {
 	}
 }
 
+func TestEquals(t *testing.T) {
+	l := LexToList("mytest", "not\n test")
+
+	if ok, msg := l[0].Equals(l[1]); ok || msg != `ID is different 46 vs 6
+Pos is different 0 vs 5
+Val is different not vs test
+Identifier is different false vs true
+Lline is different 1 vs 2
+Lpos is different 1 vs 2
+{
+  "ID": 46,
+  "Pos": 0,
+  "Val": "not",
+  "Identifier": false,
+  "Lline": 1,
+  "Lpos": 1
+}
+vs
+{
+  "ID": 6,
+  "Pos": 5,
+  "Val": "test",
+  "Identifier": true,
+  "Lline": 2,
+  "Lpos": 2
+}` {
+		t.Error("Unexpected result:", msg)
+		return
+	}
+}
+
 func TestBasicTokenLexing(t *testing.T) {
 
 	// Test empty string parsing

+ 27 - 1
lang/ecal/parser/main_test.go

@@ -10,6 +10,7 @@
 package parser
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
@@ -55,7 +56,32 @@ func UnitTestParse(name string, input string) (*ASTNode, error) {
 
 	// TODO Test pretty printing
 
-	// TODO Test AST serialization
+	// Test AST serialization
+
+	if err == nil {
+		var unmarshaledJSONObject map[string]interface{}
+
+		astString, err := json.Marshal(n.ToJSONObject())
+		if err != nil {
+			return nil, fmt.Errorf("Could not marshal AST: %v", err)
+		}
+
+		if err := json.Unmarshal(astString, &unmarshaledJSONObject); err != nil {
+			return nil, fmt.Errorf("Could not unmarshal JSON object: %v", err)
+		}
+
+		unmarshaledAST, err := ASTFromJSONObject(unmarshaledJSONObject)
+		if err != nil {
+			return nil, fmt.Errorf("Could not create AST from unmarshaled JSON object: %v", err)
+		}
+
+		// String compare the ASTs
+		if ok, msg := n.Equals(unmarshaledAST); !ok {
+			return nil, fmt.Errorf(
+				"Parsed AST is different from the unmarshaled AST.\n%v\n",
+				msg)
+		}
+	}
 
 	return n, err
 }