123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487 |
- package parser
- import (
- "fmt"
- "regexp"
- "strconv"
- "strings"
- "unicode"
- "unicode/utf8"
- "devt.de/krotik/common/stringutil"
- )
- type LexToken struct {
- ID LexTokenID
- Pos int
- Val string
- Lline int
- Lpos int
- }
- func (t LexToken) PosString() string {
- return fmt.Sprintf("Line %v, Pos %v", t.Lline, t.Lpos)
- }
- func (t LexToken) String() string {
- switch {
- case t.ID == TokenEOF:
- return "EOF"
- case t.ID == TokenError:
- return fmt.Sprintf("Error: %s (%s)", t.Val, t.PosString())
- case t.ID == TokenName:
- return fmt.Sprintf("<%s>", t.Val)
- case t.ID == TokenStringValue:
- return fmt.Sprintf("\"%s\"", t.Val)
- case t.ID == TokenIntValue:
- return fmt.Sprintf("int(%s)", t.Val)
- case t.ID == TokenFloatValue:
- return fmt.Sprintf("flt(%s)", t.Val)
- }
- return fmt.Sprintf("%s", t.Val)
- }
- var SymbolMap = map[string]LexTokenID{
- "!": TokenPunctuator,
- "$": TokenPunctuator,
- "(": TokenPunctuator,
- ")": TokenPunctuator,
- ":": TokenPunctuator,
- "=": TokenPunctuator,
- "@": TokenPunctuator,
- "[": TokenPunctuator,
- "]": TokenPunctuator,
- "{": TokenPunctuator,
- "|": TokenPunctuator,
- "}": TokenPunctuator,
-
- }
- const RuneEOF = -1
- const RuneComma = ','
- type lexFunc func() lexFunc
- type lexer struct {
- name string
- input string
- pos int
- line int
- lastnl int
- width int
- start int
- tokens chan LexToken
- }
- func Lex(name string, input string) chan LexToken {
- l := &lexer{name, input, 0, 0, 0, 0, 0, make(chan LexToken)}
- go l.run()
- return l.tokens
- }
- func LexToList(name string, input string) []LexToken {
- var tokens []LexToken
- for t := range Lex(name, input) {
- tokens = append(tokens, t)
- }
- return tokens
- }
- func (l *lexer) run() {
- if l.skipWhiteSpace() {
- for state := l.lexToken; state != nil; {
- state = state()
- if !l.skipWhiteSpace() {
- break
- }
- }
- }
- close(l.tokens)
- }
- func (l *lexer) next(peek int) rune {
- var r rune
- var w, peekw int
-
- if int(l.pos) >= len(l.input) {
- return RuneEOF
- }
-
- peeklen := 1 + peek
- if peeklen < 1 {
- peeklen = 1
- }
- for i := 0; i < peeklen; i++ {
- r, w = utf8.DecodeRuneInString(l.input[l.pos+peekw:])
- peekw += w
- }
- if peek == -1 {
- l.width = w
- l.pos += l.width
- }
- return r
- }
- func (l *lexer) hasSequence(s string) bool {
- runes := stringutil.StringToRuneSlice(s)
- for i := 0; i < len(runes); i++ {
- if l.next(i) != runes[i] {
- return false
- }
- }
- return true
- }
- func (l *lexer) startNew() {
- l.start = l.pos
- }
- func (l *lexer) emitToken(i LexTokenID, val string) {
- if l.tokens != nil {
- l.tokens <- LexToken{i, l.start, val, l.line + 1, l.start - l.lastnl + 1}
- }
- }
- func (l *lexer) lexToken() lexFunc {
- l.startNew()
- l.lexTextBlock()
- token := l.input[l.start:l.pos]
-
- if token == "#" {
- return l.skipRestOfLine()
- }
-
-
- if token == "\"" {
- return l.lexStringValue()
- }
-
- if _, ok := SymbolMap[token]; ok || token == "..." {
- l.emitToken(TokenPunctuator, token)
- return l.lexToken
- }
-
- isName, _ := regexp.MatchString("^[_A-Za-z][_0-9A-Za-z]*$", token)
- if isName {
- l.emitToken(TokenName, token)
- return l.lexToken
- }
-
- isZero, _ := regexp.MatchString("^-?0$", token)
- isInt, _ := regexp.MatchString("^-?[1-9][0-9]*$", token)
- if isZero || isInt {
- l.emitToken(TokenIntValue, token)
- return l.lexToken
- }
-
- isFloat1, _ := regexp.MatchString("^[0-9]*\\.[0-9]*$", token)
- isFloat2, _ := regexp.MatchString("^[0-9][eE][+-]?[0-9]*$", token)
- isFloat3, _ := regexp.MatchString("^[0-9]*\\.[0-9][eE][+-]?[0-9]*$", token)
- if isFloat1 || isFloat2 || isFloat3 {
- l.emitToken(TokenFloatValue, strings.ToLower(token))
- return l.lexToken
- }
-
- l.emitToken(TokenError, token)
- return l.lexToken
- }
- func (l *lexer) lexTextBlock() {
- r := l.next(0)
-
- if _, ok := SymbolMap[strings.ToLower(string(r))]; ok || r == '#' || r == '"' {
- l.next(-1)
- return
- } else if r == '.' && l.hasSequence("...") {
- l.next(-1)
- l.next(-1)
- l.next(-1)
- return
- }
- for !l.isIgnoredRune(r) {
- l.next(-1)
- r = l.next(0)
-
- if _, ok := SymbolMap[strings.ToLower(string(r))]; ok || r == '#' || r == '"' {
- return
- } else if r == '.' && l.hasSequence("...") {
- return
- }
- }
- }
- func (l *lexer) lexStringValue() lexFunc {
- var isEnd func(rune) bool
-
-
- r1 := l.next(0)
- r2 := l.next(1)
- isBlockString := r1 == '"' && r2 == '"'
- if isBlockString {
-
- l.next(-1)
- l.next(-1)
- isEnd = func(r rune) bool {
- r1 := l.next(0)
- r2 := l.next(1)
- return r == '"' && r1 == '"' && r2 == '"'
- }
- } else {
- isEnd = func(r rune) bool {
- return r == '"'
- }
- }
- r := l.next(-1)
- lLine := l.line
- lLastnl := l.lastnl
- for !isEnd(r) {
- if r == '\n' {
- lLine++
- lLastnl = l.pos
- }
- r = l.next(-1)
- if r == RuneEOF {
- l.emitToken(TokenError, "EOF inside quotes")
- return nil
- }
- }
- if !isBlockString {
- val := l.input[l.start+1 : l.pos-1]
- s, err := strconv.Unquote("\"" + val + "\"")
- if err != nil {
- l.emitToken(TokenError, "Could not interpret escape sequence: "+err.Error())
- return nil
- }
- l.emitToken(TokenStringValue, s)
- } else {
-
- l.next(-1)
- l.next(-1)
- token := l.input[l.start+3 : l.pos-3]
-
-
-
-
- token = stringutil.StripUniformIndentation(token)
- token = stringutil.TrimBlankLines(token)
- l.emitToken(TokenStringValue, token)
- }
-
- l.line = lLine
- l.lastnl = lLastnl
- return l.lexToken
- }
- func (l *lexer) isIgnoredRune(r rune) bool {
-
- return unicode.IsSpace(r) || unicode.IsControl(r) || r == RuneEOF ||
- r == RuneComma || r == '\ufeff'
- }
- func (l *lexer) skipWhiteSpace() bool {
- r := l.next(0)
- for l.isIgnoredRune(r) {
- if r == '\n' {
- l.line++
- l.lastnl = l.pos
- }
- l.next(-1)
- if r == RuneEOF {
- l.startNew()
- l.start--
- l.emitToken(TokenEOF, "")
- return false
- }
- r = l.next(0)
- }
- return true
- }
- func (l *lexer) skipRestOfLine() lexFunc {
- r := l.next(-1)
- for r != '\n' && r != RuneEOF {
- r = l.next(-1)
- }
- if r == RuneEOF {
- return nil
- }
- l.line++
- l.lastnl = l.pos - 1
- return l.lexToken
- }
|