feat: Adding proper meta data support

lang/ecal/parser/const.go

@@ -45,6 +45,15 @@ LexTokenID represents a unique lexer token ID
 */
 type LexTokenID int
 
+/*
+Available meta data types
+*/
+const (
+	MetaDataPreComment  = "MetaDataPreComment"
+	MetaDataPostComment = "MetaDataPostComment"
+	MetaDataGeneral     = "MetaDataGeneral"
+)
+
 /*
 Available lexer token types
 */

lang/ecal/parser/helper.go

@@ -21,15 +21,53 @@ import (
 // AST Nodes
 // =========
 
+/*
+MetaData is auxiliary data which can be attached to ASTs.
+*/
+type MetaData interface {
+
+	/*
+		Type returns the type of the meta data.
+	*/
+	Type() string
+
+	/*
+		Value returns the value of the meta data.
+	*/
+	Value() string
+}
+
+/*
+metaData is a minimal MetaData implementation.
+*/
+type metaData struct {
+	metatype  string
+	metavalue string
+}
+
+/*
+Type returns the type of the meta data.
+*/
+func (m *metaData) Type() string {
+	return m.metatype
+}
+
+/*
+Value returns the value of the meta data.
+*/
+func (m *metaData) Value() string {
+	return m.metavalue
+}
+
 /*
 ASTNode models a node in the AST
 */
 type ASTNode struct {
-	Name     string      // Name of the node
-	Token    *LexToken   // Lexer token of this ASTNode
-	Meta     []*LexToken // Meta data for this ASTNode (e.g. comments)
-	Children []*ASTNode  // Child nodes
-	Runtime  Runtime     // Runtime component for this ASTNode
+	Name     string     // Name of the node
+	Token    *LexToken  // Lexer token of this ASTNode
+	Meta     []MetaData // Meta data for this ASTNode (e.g. comments)
+	Children []*ASTNode // Child nodes
+	Runtime  Runtime    // Runtime component for this ASTNode
 
 	binding        int                                                             // Binding power of this node
 	nullDenotation func(p *parser, self *ASTNode) (*ASTNode, error)                // Configure token as beginning node
@@ -76,6 +114,25 @@ func (n *ASTNode) equalsPath(path string, other *ASTNode, ignoreTokenPosition bo
 		msg += fmt.Sprintf("Token is different:\n%v\n", tokenMSG)
 	}
 
+	if len(n.Meta) != len(other.Meta) {
+		res = false
+		msg = fmt.Sprintf("Number of meta data entries is different %v vs %v\n",
+			len(n.Meta), len(other.Meta))
+	} else {
+		for i, meta := range n.Meta {
+
+			// Check for different in meta entries
+
+			if meta.Type() != other.Meta[i].Type() {
+				res = false
+				msg += fmt.Sprintf("Meta data type is different %v vs %v\n", meta.Type(), other.Meta[i].Type())
+			} else if meta.Value() != other.Meta[i].Value() {
+				res = false
+				msg += fmt.Sprintf("Meta data value is different %v vs %v\n", meta.Value(), other.Meta[i].Value())
+			}
+		}
+	}
+
 	if len(n.Children) != len(other.Children) {
 		res = false
 		msg = fmt.Sprintf("Number of children is different %v vs %v\n",
@@ -135,7 +192,7 @@ func (n *ASTNode) levelString(indent int, buf *bytes.Buffer, printChildren int)
 	if len(n.Meta) > 0 {
 		buf.WriteString(" # ")
 		for i, c := range n.Meta {
-			buf.WriteString(c.Val)
+			buf.WriteString(c.Value())
 			if i < len(n.Meta)-1 {
 				buf.WriteString(" ")
 			}
@@ -166,6 +223,20 @@ func (n *ASTNode) ToJSONObject() map[string]interface{} {
 
 	ret["name"] = n.Name
 
+	lenMeta := len(n.Meta)
+
+	if lenMeta > 0 {
+		meta := make([]map[string]interface{}, lenMeta)
+		for i, metaChild := range n.Meta {
+			meta[i] = map[string]interface{}{
+				"type":  metaChild.Type(),
+				"value": metaChild.Value(),
+			}
+		}
+
+		ret["meta"] = meta
+	}
+
 	lenChildren := len(n.Children)
 
 	if lenChildren > 0 {
@@ -209,6 +280,7 @@ The following nested map structure is expected:
 	}
 */
 func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
+	var astMeta []MetaData
 	var astChildren []*ASTNode
 	var nodeID LexTokenID = TokenANY
 	var pos, line, linepos int
@@ -252,6 +324,29 @@ func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
 		linepos = 0
 	}
 
+	// Create meta data
+
+	if meta, ok := jsonAST["meta"]; ok {
+
+		if ic, ok := meta.([]interface{}); ok {
+
+			// Do a list conversion if necessary - this is necessary when we parse
+			// JSON with map[string]interface{}
+
+			metaList := make([]map[string]interface{}, len(ic))
+			for i := range ic {
+				metaList[i] = ic[i].(map[string]interface{})
+			}
+
+			meta = metaList
+		}
+
+		for _, metaChild := range meta.([]map[string]interface{}) {
+			astMeta = append(astMeta, &metaData{
+				fmt.Sprint(metaChild["type"]), fmt.Sprint(metaChild["value"])})
+		}
+	}
+
 	// Create children
 
 	if children, ok := jsonAST["children"]; ok {
@@ -289,7 +384,7 @@ func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
 		linepos,            // Lpos
 	}
 
-	return &ASTNode{fmt.Sprint(name), token, nil, astChildren, nil, 0, nil, nil}, nil
+	return &ASTNode{fmt.Sprint(name), token, astMeta, astChildren, nil, 0, nil, nil}, nil
 }
 
 // Look ahead buffer

lang/ecal/parser/helper_test.go

@@ -134,6 +134,101 @@ minus
 		return
 	}
 
+	n, err = ParseWithRuntime("", "-1 #test", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	n2, err = ParseWithRuntime("", "-1", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	if ok, msg := n.Equals(n2, false); ok || msg != `Path to difference: minus > number
+
+Number of meta data entries is different 1 vs 0
+
+AST Nodes:
+number: 1 # test
+vs
+number: 1
+` {
+		t.Error("Unexpected result: ", msg)
+		return
+	}
+
+	n, err = ParseWithRuntime("", "-1 #test", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	n2, err = ParseWithRuntime("", "-1 #wurst", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	if ok, msg := n.Equals(n2, false); ok || msg != `Path to difference: minus > number
+
+Meta data value is different test vs wurst
+
+AST Nodes:
+number: 1 # test
+vs
+number: 1 # wurst
+` {
+		t.Error("Unexpected result: ", msg)
+		return
+	}
+
+	n, err = ParseWithRuntime("", "1 #test", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	n2, err = ParseWithRuntime("", "/*test*/ 1", &DummyRuntimeProvider{})
+	if err != nil {
+		t.Error("Cannot parse test AST:", err)
+		return
+	}
+
+	if ok, msg := n.Equals(n2, false); ok || msg != `Path to difference: number
+
+Token is different:
+Pos is different 0 vs 9
+Lpos is different 1 vs 10
+{
+  "ID": 6,
+  "Pos": 0,
+  "Val": "1",
+  "Identifier": false,
+  "Lline": 1,
+  "Lpos": 1
+}
+vs
+{
+  "ID": 6,
+  "Pos": 9,
+  "Val": "1",
+  "Identifier": false,
+  "Lline": 1,
+  "Lpos": 10
+}
+Meta data type is different MetaDataPostComment vs MetaDataPreComment
+
+AST Nodes:
+number: 1 # test
+vs
+number: 1 # test
+` {
+		t.Error("Unexpected result: ", msg)
+		return
+	}
+
 	// Test building an AST from an invalid
 
 	if _, err := ASTFromJSONObject(map[string]interface{}{

lang/ecal/parser/lexer.go

@@ -148,6 +148,27 @@ func (t LexToken) String() string {
 	return fmt.Sprintf("%s%q", prefix, t.Val)
 }
 
+// Meta data interface
+
+/*
+Type returns the meta data type.
+*/
+func (t LexToken) Type() string {
+	if t.ID == TokenPRECOMMENT {
+		return MetaDataPreComment
+	} else if t.ID == TokenPOSTCOMMENT {
+		return MetaDataPostComment
+	}
+	return MetaDataGeneral
+}
+
+/*
+Value returns the meta data value.
+*/
+func (t LexToken) Value() string {
+	return t.Val
+}
+
 /*
 KeywordMap is a map of keywords - these require spaces between them
 */

lang/ecal/parser/lexer_test.go

@@ -18,7 +18,9 @@ func TestNextItem(t *testing.T) {
 
 	l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, make(chan LexToken)}
 
-	if r := l.next(1); r != '1' {
+	r := l.next(1)
+
+	if r != '1' {
 		t.Errorf("Unexpected token: %q", r)
 		return
 	}
@@ -62,6 +64,11 @@ func TestNextItem(t *testing.T) {
 func TestEquals(t *testing.T) {
 	l := LexToList("mytest", "not\n test")
 
+	if mt := l[0].Type(); mt != "MetaDataGeneral" {
+		t.Error("Unexpected meta type:", mt)
+		return
+	}
+
 	if ok, msg := l[0].Equals(l[1], false); ok || msg != `ID is different 47 vs 7
 Pos is different 0 vs 5
 Val is different not vs test

lang/ecal/parser/parser.go

@@ -232,8 +232,8 @@ func (p *parser) run(rightBinding int) (*ASTNode, error) {
 next retrieves the next lexer token.
 */
 func (p *parser) next() (*ASTNode, error) {
-	var preComments []*LexToken
-	var postComments []*LexToken
+	var preComments []MetaData
+	var postComments []MetaData
 
 	token, more := p.tokens.Next()
 

lang/ecal/parser/parser_main_test.go

@@ -31,6 +31,16 @@ func TestCommentParsing(t *testing.T) {
 		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
 		return
 	}
+
+	input = `/* foo */ 1 # foo bar`
+	expectedOutput = `
+number: 1 #  foo   foo bar
+`[1:]
+
+	if res, err := UnitTestParse("mytest", input); err != nil || fmt.Sprint(res) != expectedOutput {
+		t.Error("Unexpected parser output:\n", res, "expected was:\n", expectedOutput, "Error:", err)
+		return
+	}
 }
 
 func TestSimpleExpressionParsing(t *testing.T) {

lang/ecal/parser/prettyprinter.go

@@ -147,7 +147,21 @@ func PrettyPrint(ast *ASTNode) (string, error) {
 
 		errorutil.AssertOk(temp.Execute(&buf, tempParam))
 
-		return buf.String(), nil
+		ret := buf.String()
+
+		// Add meta data
+
+		if len(ast.Meta) > 0 {
+			for _, meta := range ast.Meta {
+				if meta.Type() == MetaDataPreComment {
+					ret = fmt.Sprintf("/*%v*/ %v", meta.Value(), ret)
+				} else if meta.Type() == MetaDataPostComment {
+					ret = fmt.Sprintf("%v #%v", ret, meta.Value())
+				}
+			}
+		}
+
+		return ret, nil
 	}
 
 	return visit(ast, 0)