fix: Pretty printer preserves some newlines to structure code.

interpreter/debug_test.go

@@ -331,6 +331,7 @@ log("test3")
             "Val": "raise",
             "Identifier": true,
             "AllowEscapes": false,
+            "PrefixNewlines": 1,
             "Lsource": "ECALEvalTest",
             "Lline": 3,
             "Lpos": 2
@@ -350,6 +351,7 @@ log("test3")
                     "Val": "foo",
                     "Identifier": false,
                     "AllowEscapes": true,
+                    "PrefixNewlines": 0,
                     "Lsource": "ECALEvalTest",
                     "Lline": 3,
                     "Lpos": 8

parser/helper.go

@@ -313,6 +313,7 @@ func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
 	value, _ := getVal("value", "")
 	identifier, _ := getVal("identifier", false)
 	allowescapes, _ := getVal("allowescapes", false)
+	_, prefixnl := getVal("prefixnewlines", "")
 	_, pos := getVal("pos", "")
 	_, line := getVal("line", "")
 	_, linepos := getVal("linepos", "")
@@ -375,6 +376,7 @@ func ASTFromJSONObject(jsonAST map[string]interface{}) (*ASTNode, error) {
 		fmt.Sprint(value),    // Val
 		identifier == true,   // Identifier
 		allowescapes == true, // AllowEscapes
+		prefixnl,             // PrefixNewlines
 		fmt.Sprint(source),   // Lsource
 		line,                 // Lline
 		linepos,              // Lpos

parser/helper_test.go

@@ -40,6 +40,7 @@ Lpos is different 3 vs 2
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "PrefixNewlines": 0,
   "Lsource": "test1",
   "Lline": 1,
   "Lpos": 3
@@ -51,6 +52,7 @@ vs
   "Val": "2",
   "Identifier": false,
   "AllowEscapes": false,
+  "PrefixNewlines": 0,
   "Lsource": "test2",
   "Lline": 1,
   "Lpos": 2
@@ -90,6 +92,7 @@ Identifier is different false vs true
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "PrefixNewlines": 0,
   "Lsource": "test1",
   "Lline": 1,
   "Lpos": 2
@@ -101,6 +104,7 @@ vs
   "Val": "a",
   "Identifier": true,
   "AllowEscapes": false,
+  "PrefixNewlines": 0,
   "Lsource": "test2",
   "Lline": 1,
   "Lpos": 2
@@ -222,6 +226,7 @@ Lpos is different 1 vs 10
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "PrefixNewlines": 0,
   "Lsource": "",
   "Lline": 1,
   "Lpos": 1
@@ -233,6 +238,7 @@ vs
   "Val": "1",
   "Identifier": false,
   "AllowEscapes": false,
+  "PrefixNewlines": 0,
   "Lsource": "",
   "Lline": 1,
   "Lpos": 10

parser/lexer.go

@@ -35,14 +35,15 @@ var numberPattern = regexp.MustCompile("^[0-9].*$")
 LexToken represents a token which is returned by the lexer.
 */
 type LexToken struct {
-	ID           LexTokenID // Token kind
-	Pos          int        // Starting position (in bytes)
-	Val          string     // Token value
-	Identifier   bool       // Flag if the value is an identifier (not quoted and not a number)
-	AllowEscapes bool       // Flag if the value did interpret escape charaters
-	Lsource      string     // Input source label (e.g. filename)
-	Lline        int        // Line in the input this token appears
-	Lpos         int        // Position in the input line this token appears
+	ID             LexTokenID // Token kind
+	Pos            int        // Starting position (in bytes)
+	Val            string     // Token value
+	Identifier     bool       // Flag if the value is an identifier (not quoted and not a number)
+	AllowEscapes   bool       // Flag if the value did interpret escape charaters
+	PrefixNewlines int        // Number of newlines which precede this token
+	Lsource        string     // Input source label (e.g. filename)
+	Lline          int        // Line in the input this token appears
+	Lpos           int        // Position in the input line this token appears
 }
 
 /*
@@ -55,6 +56,7 @@ func NewLexTokenInstance(t LexToken) *LexToken {
 		t.Val,
 		t.Identifier,
 		t.AllowEscapes,
+		t.PrefixNewlines,
 		t.Lsource,
 		t.Lline,
 		t.Lpos,
@@ -321,21 +323,22 @@ type lexFunc func(*lexer) lexFunc
 Lexer data structure
 */
 type lexer struct {
-	name   string        // Name to identify the input
-	input  string        // Input string of the lexer
-	pos    int           // Current rune pointer
-	line   int           // Current line pointer
-	lastnl int           // Last newline position
-	width  int           // Width of last rune
-	start  int           // Start position of the current red token
-	tokens chan LexToken // Channel for lexer output
+	name           string        // Name to identify the input
+	input          string        // Input string of the lexer
+	pos            int           // Current rune pointer
+	line           int           // Current line pointer
+	lastnl         int           // Last newline position
+	skippedNewline int           // Number of skipped newlines
+	width          int           // Width of last rune
+	start          int           // Start position of the current red token
+	tokens         chan LexToken // Channel for lexer output
 }
 
 /*
 Lex lexes a given input. Returns a channel which contains tokens.
 */
 func Lex(name string, input string) chan LexToken {
-	l := &lexer{name, input, 0, 0, 0, 0, 0, make(chan LexToken)}
+	l := &lexer{name, input, 0, 0, 0, 0, 0, 0, make(chan LexToken)}
 	go l.run()
 	return l.tokens
 }
@@ -428,7 +431,7 @@ func (l *lexer) emitToken(t LexTokenID) {
 	}
 
 	if l.tokens != nil {
-		l.tokens <- LexToken{t, l.start, l.input[l.start:l.pos], false, false, l.name,
+		l.tokens <- LexToken{t, l.start, l.input[l.start:l.pos], false, false, l.skippedNewline, l.name,
 			l.line + 1, l.start - l.lastnl + 1}
 	}
 }
@@ -438,7 +441,8 @@ emitTokenAndValue passes a token with a given value back to the client.
 */
 func (l *lexer) emitTokenAndValue(t LexTokenID, val string, identifier bool, allowEscapes bool) {
 	if l.tokens != nil {
-		l.tokens <- LexToken{t, l.start, val, identifier, allowEscapes, l.name, l.line + 1, l.start - l.lastnl + 1}
+		l.tokens <- LexToken{t, l.start, val, identifier, allowEscapes, l.skippedNewline,
+			l.name, l.line + 1, l.start - l.lastnl + 1}
 	}
 }
 
@@ -447,7 +451,8 @@ emitError passes an error token back to the client.
 */
 func (l *lexer) emitError(msg string) {
 	if l.tokens != nil {
-		l.tokens <- LexToken{TokenError, l.start, msg, false, false, l.name, l.line + 1, l.start - l.lastnl + 1}
+		l.tokens <- LexToken{TokenError, l.start, msg, false, false, l.skippedNewline,
+			l.name, l.line + 1, l.start - l.lastnl + 1}
 	}
 }
 
@@ -460,10 +465,12 @@ reaches EOF while skipping whitespaces.
 */
 func skipWhiteSpace(l *lexer) bool {
 	r := l.next(0)
+	l.skippedNewline = 0
 
 	for unicode.IsSpace(r) || unicode.IsControl(r) || r == RuneEOF {
 		if r == '\n' {
 			l.line++
+			l.skippedNewline++
 			l.lastnl = l.pos
 		}
 		r = l.next(0)

parser/lexer_test.go

@@ -17,7 +17,7 @@ import (
 
 func TestNextItem(t *testing.T) {
 
-	l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, make(chan LexToken)}
+	l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, 0, make(chan LexToken)}
 
 	r := l.next(1)
 
@@ -82,6 +82,7 @@ Lpos is different 1 vs 2
   "Val": "not",
   "Identifier": false,
   "AllowEscapes": false,
+  "PrefixNewlines": 0,
   "Lsource": "mytest",
   "Lline": 1,
   "Lpos": 1
@@ -93,6 +94,7 @@ vs
   "Val": "test",
   "Identifier": true,
   "AllowEscapes": false,
+  "PrefixNewlines": 1,
   "Lsource": "mytest",
   "Lline": 2,
   "Lpos": 2

parser/prettyprinter.go

@@ -17,6 +17,7 @@ import (
 	"strconv"
 	"strings"
 	"text/template"
+	"unicode"
 
 	"devt.de/krotik/common/errorutil"
 	"devt.de/krotik/common/stringutil"
@@ -238,35 +239,10 @@ func PrettyPrint(ast *ASTNode) (string, error) {
 ppPostProcessing applies post processing rules.
 */
 func ppPostProcessing(ast *ASTNode, path []*ASTNode, ppString string) string {
-	ret := ppString
 
 	// Add meta data
 
-	if len(ast.Meta) > 0 {
-
-		for _, meta := range ast.Meta {
-			metaValue := meta.Value()
-			if meta.Type() == MetaDataPreComment {
-				var buf bytes.Buffer
-
-				scanner := bufio.NewScanner(strings.NewReader(metaValue))
-				for scanner.Scan() {
-					buf.WriteString(fmt.Sprintf(" %v\n", strings.TrimSpace(scanner.Text())))
-				}
-				buf.Truncate(buf.Len() - 1) // Remove the last newline
-
-				if strings.Index(buf.String(), "\n") == -1 {
-					buf.WriteString(" ")
-				}
-
-				ret = fmt.Sprintf("/*%v*/\n%v", buf.String(), ret)
-
-			} else if meta.Type() == MetaDataPostComment {
-				metaValue = strings.TrimSpace(strings.ReplaceAll(metaValue, "\n", ""))
-				ret = fmt.Sprintf("%v # %v", ret, metaValue)
-			}
-		}
-	}
+	ret := ppMetaData(ast, path, ppString)
 
 	// Apply indentation
 
@@ -316,6 +292,77 @@ func ppPostProcessing(ast *ASTNode, path []*ASTNode, ppString string) string {
 		}
 	}
 
+	if ast.Token != nil {
+
+		// Calculate number of extra newlines which should be prefixed the default
+		// pretty printer assumes a single one which produces very compact code
+		// the previous formatting might give a hint where to add extra newlines.
+		// The pretty printer only ever adds a maximum of 1 additional line per
+		// statement
+
+		if ast.Token.PrefixNewlines-1 > 0 {
+			ret = fmt.Sprintf("\n%v", ret)
+		}
+	}
+
+	// Remove all trailing spaces
+
+	newlineSplit := strings.Split(ret, "\n")
+
+	for i, s := range newlineSplit {
+		newlineSplit[i] = strings.TrimRightFunc(s, unicode.IsSpace)
+	}
+
+	return strings.Join(newlineSplit, "\n")
+}
+
+/*
+ppMetaData pretty prints comments.
+*/
+func ppMetaData(ast *ASTNode, path []*ASTNode, ppString string) string {
+	ret := ppString
+
+	if len(ast.Meta) > 0 {
+
+		for _, meta := range ast.Meta {
+			metaValue := meta.Value()
+			if meta.Type() == MetaDataPreComment {
+				var buf bytes.Buffer
+
+				scanner := bufio.NewScanner(strings.NewReader(metaValue))
+				for scanner.Scan() {
+					buf.WriteString(fmt.Sprintf(" %v\n", strings.TrimSpace(scanner.Text())))
+				}
+
+				if ast.Token.Lpos != 1 || strings.Index(metaValue, "\n") == -1 {
+
+					// Remove the last newline if we are not on the root level
+					// or we didn't have any newlines in the original comment
+
+					buf.Truncate(buf.Len() - 1)
+				}
+
+				if strings.Index(buf.String(), "\n") == -1 {
+
+					// If the pretty printed comment does not have any newlines
+					// Add at least a space at the end
+
+					buf.WriteString(" ")
+				}
+
+				ret = fmt.Sprintf("/*%v*/\n%v", buf.String(), ret)
+
+				if ast.Token.Lline > 1 {
+					ret = fmt.Sprintf("\n%v", ret)
+				}
+
+			} else if meta.Type() == MetaDataPostComment {
+				metaValue = strings.TrimSpace(strings.ReplaceAll(metaValue, "\n", ""))
+				ret = fmt.Sprintf("%v # %v", ret, metaValue)
+			}
+		}
+	}
+
 	return ret
 }
 

parser/prettyprinter_test.go

@@ -453,18 +453,19 @@ b:=1
 `
 
 	if err := UnitTestPrettyPrinting(input, "",
-		`/* 
- 
+		`/*
+
  Some initial comment
- 
+
  bla
- */
+
+*/
 a := 1
 func aaa() {
     mutex myresource {
         globalResource := "new value"
     }
-    
+
     func myfunc(a, b, c=1) {
         a := 1 + 1 # Test
     }
@@ -477,26 +478,31 @@ func aaa() {
     ]
     a := 1
     b := 1
+
     /* Foo */
     Foo := {
         "super" : [Bar],
-        /* 
+
+        /*
          * Object IDs
          */
         "id" : 0 # aaaa,
         "idx" : 0,
+
         /* Constructor */
         "init" : func (id) {
             super[0]()
             this.id := id
         },
-        /* 
+
+        /*
          Return the object ID
          */
         "getId" : func () {
             return this.idx
         },
-        /* 
+
+        /*
          Set the object ID
          */
         "setId" : func (id) {
@@ -527,3 +533,101 @@ b := 1`); err != nil {
 		return
 	}
 }
+
+func TestSpacing(t *testing.T) {
+	input := `
+	
+	import "./templates.ecal" as templates
+	
+	
+a := 1
+a := 2
+
+
+/*
+  SomeSink
+*/
+sink SomeSink
+  kindmatch   ["foo",2]
+  statematch {"a":1,"b":1,"c":1,"d":1}
+scopematch []
+suppresses ["abs"]
+priority 0
+{
+log("1223")
+log("1223")
+
+t = r"Foo bar
+    {{1+2}}   
+    aaa"
+
+
+func foo (z=[1,2,3,4,5]) {
+a := 1
+b := 2
+
+c := 1
+d := 1
+}
+log("1223")
+try {
+x := [1,2,3,4]
+    raise("test 12", null, [1,2,3])
+} except e {
+p := 1
+}
+}
+`
+
+	if err := UnitTestPrettyPrinting(input, "",
+		`import "./templates.ecal" as templates
+
+a := 1
+a := 2
+
+/*
+ SomeSink
+*/
+sink SomeSink
+    kindmatch ["foo", 2]
+    statematch {
+        "a" : 1,
+        "b" : 1,
+        "c" : 1,
+        "d" : 1
+    }
+    scopematch []
+    suppresses ["abs"]
+    priority 0
+{
+    log("1223")
+    log("1223")
+
+    t="Foo bar\n    {{1+2}}   \n    aaa"
+
+    func foo(z=[
+        1,
+        2,
+        3,
+        4,
+        5
+    ]) {
+        a := 1
+        b := 2
+
+        c := 1
+        d := 1
+    }
+    log("1223")
+    try {
+        x := [1, 2, 3, 4]
+        raise("test 12", null, [1, 2, 3])
+    } except e {
+        p := 1
+    }
+}`); err != nil {
+		t.Error(err)
+		return
+	}
+
+}

util/error_test.go

@@ -73,6 +73,7 @@ raise(c) (bar2:1)
       "Val": ":=",
       "Identifier": false,
       "AllowEscapes": false,
+      "PrefixNewlines": 0,
       "Lsource": "foo",
       "Lline": 1,
       "Lpos": 2
@@ -87,6 +88,7 @@ raise(c) (bar2:1)
           "Val": "a",
           "Identifier": true,
           "AllowEscapes": false,
+          "PrefixNewlines": 0,
           "Lsource": "foo",
           "Lline": 1,
           "Lpos": 1
@@ -103,6 +105,7 @@ raise(c) (bar2:1)
           "Val": "1",
           "Identifier": false,
           "AllowEscapes": false,
+          "PrefixNewlines": 0,
           "Lsource": "foo",
           "Lline": 1,
           "Lpos": 4
@@ -124,6 +127,7 @@ raise(c) (bar2:1)
         "Val": "print",
         "Identifier": true,
         "AllowEscapes": false,
+        "PrefixNewlines": 0,
         "Lsource": "bar1",
         "Lline": 1,
         "Lpos": 1
@@ -143,6 +147,7 @@ raise(c) (bar2:1)
                 "Val": "b",
                 "Identifier": true,
                 "AllowEscapes": false,
+                "PrefixNewlines": 0,
                 "Lsource": "bar1",
                 "Lline": 1,
                 "Lpos": 7
@@ -165,6 +170,7 @@ raise(c) (bar2:1)
         "Val": "raise",
         "Identifier": true,
         "AllowEscapes": false,
+        "PrefixNewlines": 0,
         "Lsource": "bar2",
         "Lline": 1,
         "Lpos": 1
@@ -184,6 +190,7 @@ raise(c) (bar2:1)
                 "Val": "c",
                 "Identifier": true,
                 "AllowEscapes": false,
+                "PrefixNewlines": 0,
                 "Lsource": "bar2",
                 "Lline": 1,
                 "Lpos": 7
@@ -206,6 +213,7 @@ raise(c) (bar2:1)
         "Val": "+",
         "Identifier": false,
         "AllowEscapes": false,
+        "PrefixNewlines": 0,
         "Lsource": "bar3",
         "Lline": 1,
         "Lpos": 3
@@ -220,6 +228,7 @@ raise(c) (bar2:1)
             "Val": "1",
             "Identifier": false,
             "AllowEscapes": false,
+            "PrefixNewlines": 0,
             "Lsource": "bar3",
             "Lline": 1,
             "Lpos": 1
@@ -236,6 +245,7 @@ raise(c) (bar2:1)
             "Val": "d",
             "Identifier": true,
             "AllowEscapes": false,
+            "PrefixNewlines": 0,
             "Lsource": "bar3",
             "Lline": 1,
             "Lpos": 5
@@ -273,6 +283,7 @@ raise(c) (bar2:1)
       "Val": ":=",
       "Identifier": false,
       "AllowEscapes": false,
+      "PrefixNewlines": 0,
       "Lsource": "foo",
       "Lline": 1,
       "Lpos": 2
@@ -287,6 +298,7 @@ raise(c) (bar2:1)
           "Val": "a",
           "Identifier": true,
           "AllowEscapes": false,
+          "PrefixNewlines": 0,
           "Lsource": "foo",
           "Lline": 1,
           "Lpos": 1
@@ -303,6 +315,7 @@ raise(c) (bar2:1)
           "Val": "1",
           "Identifier": false,
           "AllowEscapes": false,
+          "PrefixNewlines": 0,
           "Lsource": "foo",
           "Lline": 1,
           "Lpos": 4
@@ -324,6 +337,7 @@ raise(c) (bar2:1)
         "Val": "print",
         "Identifier": true,
         "AllowEscapes": false,
+        "PrefixNewlines": 0,
         "Lsource": "bar1",
         "Lline": 1,
         "Lpos": 1
@@ -343,6 +357,7 @@ raise(c) (bar2:1)
                 "Val": "b",
                 "Identifier": true,
                 "AllowEscapes": false,
+                "PrefixNewlines": 0,
                 "Lsource": "bar1",
                 "Lline": 1,
                 "Lpos": 7
@@ -365,6 +380,7 @@ raise(c) (bar2:1)
         "Val": "raise",
         "Identifier": true,
         "AllowEscapes": false,
+        "PrefixNewlines": 0,
         "Lsource": "bar2",
         "Lline": 1,
         "Lpos": 1
@@ -384,6 +400,7 @@ raise(c) (bar2:1)
                 "Val": "c",
                 "Identifier": true,
                 "AllowEscapes": false,
+                "PrefixNewlines": 0,
                 "Lsource": "bar2",
                 "Lline": 1,
                 "Lpos": 7
@@ -406,6 +423,7 @@ raise(c) (bar2:1)
         "Val": "+",
         "Identifier": false,
         "AllowEscapes": false,
+        "PrefixNewlines": 0,
         "Lsource": "bar3",
         "Lline": 1,
         "Lpos": 3
@@ -420,6 +438,7 @@ raise(c) (bar2:1)
             "Val": "1",
             "Identifier": false,
             "AllowEscapes": false,
+            "PrefixNewlines": 0,
             "Lsource": "bar3",
             "Lline": 1,
             "Lpos": 1
@@ -436,6 +455,7 @@ raise(c) (bar2:1)
             "Val": "d",
             "Identifier": true,
             "AllowEscapes": false,
+            "PrefixNewlines": 0,
             "Lsource": "bar3",
             "Lline": 1,
             "Lpos": 5