|
@@ -35,14 +35,15 @@ var numberPattern = regexp.MustCompile("^[0-9].*$")
|
|
|
LexToken represents a token which is returned by the lexer.
|
|
|
*/
|
|
|
type LexToken struct {
|
|
|
- ID LexTokenID
|
|
|
- Pos int
|
|
|
- Val string
|
|
|
- Identifier bool
|
|
|
- AllowEscapes bool
|
|
|
- Lsource string
|
|
|
- Lline int
|
|
|
- Lpos int
|
|
|
+ ID LexTokenID
|
|
|
+ Pos int
|
|
|
+ Val string
|
|
|
+ Identifier bool
|
|
|
+ AllowEscapes bool
|
|
|
+ PrefixNewlines int
|
|
|
+ Lsource string
|
|
|
+ Lline int
|
|
|
+ Lpos int
|
|
|
}
|
|
|
|
|
|
|
|
@@ -55,6 +56,7 @@ func NewLexTokenInstance(t LexToken) *LexToken {
|
|
|
t.Val,
|
|
|
t.Identifier,
|
|
|
t.AllowEscapes,
|
|
|
+ t.PrefixNewlines,
|
|
|
t.Lsource,
|
|
|
t.Lline,
|
|
|
t.Lpos,
|
|
@@ -321,21 +323,22 @@ type lexFunc func(*lexer) lexFunc
|
|
|
Lexer data structure
|
|
|
*/
|
|
|
type lexer struct {
|
|
|
- name string
|
|
|
- input string
|
|
|
- pos int
|
|
|
- line int
|
|
|
- lastnl int
|
|
|
- width int
|
|
|
- start int
|
|
|
- tokens chan LexToken
|
|
|
+ name string
|
|
|
+ input string
|
|
|
+ pos int
|
|
|
+ line int
|
|
|
+ lastnl int
|
|
|
+ skippedNewline int
|
|
|
+ width int
|
|
|
+ start int
|
|
|
+ tokens chan LexToken
|
|
|
}
|
|
|
|
|
|
|
|
|
Lex lexes a given input. Returns a channel which contains tokens.
|
|
|
*/
|
|
|
func Lex(name string, input string) chan LexToken {
|
|
|
- l := &lexer{name, input, 0, 0, 0, 0, 0, make(chan LexToken)}
|
|
|
+ l := &lexer{name, input, 0, 0, 0, 0, 0, 0, make(chan LexToken)}
|
|
|
go l.run()
|
|
|
return l.tokens
|
|
|
}
|
|
@@ -428,7 +431,7 @@ func (l *lexer) emitToken(t LexTokenID) {
|
|
|
}
|
|
|
|
|
|
if l.tokens != nil {
|
|
|
- l.tokens <- LexToken{t, l.start, l.input[l.start:l.pos], false, false, l.name,
|
|
|
+ l.tokens <- LexToken{t, l.start, l.input[l.start:l.pos], false, false, l.skippedNewline, l.name,
|
|
|
l.line + 1, l.start - l.lastnl + 1}
|
|
|
}
|
|
|
}
|
|
@@ -438,7 +441,8 @@ emitTokenAndValue passes a token with a given value back to the client.
|
|
|
*/
|
|
|
func (l *lexer) emitTokenAndValue(t LexTokenID, val string, identifier bool, allowEscapes bool) {
|
|
|
if l.tokens != nil {
|
|
|
- l.tokens <- LexToken{t, l.start, val, identifier, allowEscapes, l.name, l.line + 1, l.start - l.lastnl + 1}
|
|
|
+ l.tokens <- LexToken{t, l.start, val, identifier, allowEscapes, l.skippedNewline,
|
|
|
+ l.name, l.line + 1, l.start - l.lastnl + 1}
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -447,7 +451,8 @@ emitError passes an error token back to the client.
|
|
|
*/
|
|
|
func (l *lexer) emitError(msg string) {
|
|
|
if l.tokens != nil {
|
|
|
- l.tokens <- LexToken{TokenError, l.start, msg, false, false, l.name, l.line + 1, l.start - l.lastnl + 1}
|
|
|
+ l.tokens <- LexToken{TokenError, l.start, msg, false, false, l.skippedNewline,
|
|
|
+ l.name, l.line + 1, l.start - l.lastnl + 1}
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -460,10 +465,12 @@ reaches EOF while skipping whitespaces.
|
|
|
*/
|
|
|
func skipWhiteSpace(l *lexer) bool {
|
|
|
r := l.next(0)
|
|
|
+ l.skippedNewline = 0
|
|
|
|
|
|
for unicode.IsSpace(r) || unicode.IsControl(r) || r == RuneEOF {
|
|
|
if r == '\n' {
|
|
|
l.line++
|
|
|
+ l.skippedNewline++
|
|
|
l.lastnl = l.pos
|
|
|
}
|
|
|
r = l.next(0)
|