lexer_test.go 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271
  1. /*
  2. * Public Domain Software
  3. *
  4. * I (Matthias Ladkau) am the author of the source code in this file.
  5. * I have placed the source code in this file in the public domain.
  6. *
  7. * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
  8. */
  9. package parser
  10. import (
  11. "fmt"
  12. "testing"
  13. )
  14. func TestNextItem(t *testing.T) {
  15. l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, make(chan LexToken)}
  16. if r := l.next(1); r != '1' {
  17. t.Errorf("Unexpected token: %q", r)
  18. return
  19. }
  20. if r := l.next(0); r != '1' {
  21. t.Errorf("Unexpected token: %q", r)
  22. return
  23. }
  24. if r := l.next(0); r != '2' {
  25. t.Errorf("Unexpected token: %q", r)
  26. return
  27. }
  28. if r := l.next(1); r != '3' {
  29. t.Errorf("Unexpected token: %q", r)
  30. return
  31. }
  32. if r := l.next(2); r != '4' {
  33. t.Errorf("Unexpected token: %q", r)
  34. return
  35. }
  36. if r := l.next(0); r != '3' {
  37. t.Errorf("Unexpected token: %q", r)
  38. return
  39. }
  40. if r := l.next(0); r != '4' {
  41. t.Errorf("Unexpected token: %q", r)
  42. return
  43. }
  44. if r := l.next(0); r != RuneEOF {
  45. t.Errorf("Unexpected token: %q", r)
  46. return
  47. }
  48. }
  49. func TestBasicTokenLexing(t *testing.T) {
  50. // Test empty string parsing
  51. if res := fmt.Sprint(LexToList("mytest", " \t ")); res != "[EOF]" {
  52. t.Error("Unexpected lexer result:\n ", res)
  53. return
  54. }
  55. // Test arithmetics
  56. input := `name := a + 1 and (ver+x!=1) * 5 > name2`
  57. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  58. `["name" := "a" + v:"1" <AND> ( "ver" + "x" != v:"1" ) * v:"5" > "name2" EOF]` {
  59. t.Error("Unexpected lexer result:\n ", res)
  60. return
  61. }
  62. input = `test := not a * 1.3 or (12 / aa) * 5 DIV 3 % 1 > true`
  63. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  64. `["test" := <NOT> "a" * v:"1.3" <OR> ( v:"12" / "aa" ) * v:"5" "div" v:"3" % v:"1" > <TRUE> EOF]` {
  65. t.Error("Unexpected lexer result:\n ", res)
  66. return
  67. }
  68. input = `-1.234560e+02+5+2.123 // 1`
  69. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  70. `[- v:"1.234560e+02" + v:"5" + v:"2.123" // v:"1" EOF]` {
  71. t.Error("Unexpected lexer result:\n ", res)
  72. return
  73. }
  74. // Test invalid identifier
  75. input = `5test`
  76. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  77. `[v:"5" "test" EOF]` {
  78. t.Error("Unexpected lexer result:\n ", res)
  79. return
  80. }
  81. input = `@test`
  82. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  83. `[Error: Cannot parse identifier '@test'. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character (Line 1, Pos 1) EOF]` {
  84. t.Error("Unexpected lexer result:\n ", res)
  85. return
  86. }
  87. }
  88. func TestAssignmentLexing(t *testing.T) {
  89. input := `name := a + 1`
  90. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  91. `["name" := "a" + v:"1" EOF]` {
  92. t.Error("Unexpected lexer result:", res)
  93. return
  94. }
  95. input = `name := a.a + a.b`
  96. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  97. `["name" := "a" . "a" + "a" . "b" EOF]` {
  98. t.Error("Unexpected lexer result:", res)
  99. return
  100. }
  101. input = `name:=a[1] + b["d"] + c[a]`
  102. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  103. `["name" := "a" [ v:"1" ] + "b" [ "d" ] + "c" [ "a" ] EOF]` {
  104. t.Error("Unexpected lexer result:", res)
  105. return
  106. }
  107. }
  108. func TestBlockLexing(t *testing.T) {
  109. input := `
  110. if a == 1 {
  111. print("xxx")
  112. } elif b > 2 {
  113. print("yyy")
  114. } else {
  115. print("zzz")
  116. }
  117. `
  118. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  119. `[<IF> "a" == v:"1" { "print" ( "xxx" ) } <ELIF> "b" > v:"2" { "print" ( "yyy" ) } <ELSE> { "print" ( "zzz" ) } EOF]` {
  120. t.Error("Unexpected lexer result:", res)
  121. return
  122. }
  123. input = `
  124. for a, b in enum(blist) {
  125. do(a)
  126. }
  127. `
  128. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  129. `[<FOR> "a" , "b" <IN> "enum" ( "blist" ) { "do" ( "a" ) } EOF]` {
  130. t.Error("Unexpected lexer result:", res)
  131. return
  132. }
  133. input = `
  134. for true {
  135. x := "1"
  136. break; continue
  137. }
  138. `
  139. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  140. `[<FOR> <TRUE> { "x" := "1" <BREAK> ; <CONTINUE> } EOF]` {
  141. t.Error("Unexpected lexer result:", res)
  142. return
  143. }
  144. }
  145. func TestStringLexing(t *testing.T) {
  146. // Test unclosed quotes
  147. input := `name "test bla`
  148. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading string value (unclosed quotes) (Line 1, Pos 6) EOF]` {
  149. t.Error("Unexpected lexer result:", res)
  150. return
  151. }
  152. input = `name "test" 'bla'`
  153. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" "test" "bla" EOF]` {
  154. t.Error("Unexpected lexer result:", res)
  155. return
  156. }
  157. input = `name "te
  158. st" 'bla'`
  159. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: invalid syntax while parsing string (Line 1, Pos 6)]` {
  160. t.Error("Unexpected lexer result:", res)
  161. return
  162. }
  163. input = `name r"te
  164. st" 'bla'`
  165. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" "te\n\tst" "bla" EOF]` {
  166. t.Error("Unexpected lexer result:", res)
  167. return
  168. }
  169. // Parsing with escape sequences
  170. input = `"test\n\ttest" '\nfoo\u0028bar' "test{foo}.5w3f"`
  171. if res := LexToList("mytest", input); fmt.Sprint(res) != `["test\n\ttest" "\nfoo(bar" "test{foo}.5w3f" EOF]` {
  172. t.Error("Unexpected lexer result:", res)
  173. return
  174. }
  175. }
  176. func TestCommentLexing(t *testing.T) {
  177. input := `name /* foo
  178. bar
  179. x*/ 'b/* - */la' /*test*/`
  180. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" c:' foo
  181. bar
  182. x' "b/* - */la" c:'test' EOF]` {
  183. t.Error("Unexpected lexer result:", res)
  184. return
  185. }
  186. input = `name /* foo
  187. bar`
  188. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading comment (Line 1, Pos 8) EOF]` {
  189. t.Error("Unexpected lexer result:", res)
  190. return
  191. }
  192. input = `foo
  193. 1+ 2 # Some comment
  194. bar`
  195. if res := LexToList("mytest", input); fmt.Sprint(res) != `["foo" v:"1" + v:"2" c:' Some comment' "bar" EOF]` {
  196. t.Error("Unexpected lexer result:", res)
  197. return
  198. }
  199. input = `1+ 2 # Some comment`
  200. if res := LexToList("mytest", input); fmt.Sprint(res) != `[v:"1" + v:"2" c:' Some commen' EOF]` {
  201. t.Error("Unexpected lexer result:", res)
  202. return
  203. }
  204. }
  205. func TestSinkLexing(t *testing.T) {
  206. input := `sink "mysink"
  207. r"
  208. A comment describing the sink.
  209. "
  210. kindmatch [ foo.bar.* ],
  211. scopematch [ "data.read", "data.write" ],
  212. statematch { a : 1, b : NULL },
  213. priority 0,
  214. suppresses [ "myothersink" ]
  215. {
  216. a := 1
  217. }`
  218. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<SINK> "mysink" "\nA comment"... <KINDMATCH> `+
  219. `[ "foo" . "bar" . * ] , <SCOPEMATCH> [ "data.read" , "data.write" ] , <STATEMATCH> `+
  220. `{ "a" : v:"1" , "b" : <NULL> } , <PRIORITY> v:"0" , <SUPPRESSES> [ "myothersink" ] `+
  221. `{ "a" := v:"1" } EOF]` {
  222. t.Error("Unexpected lexer result:", res)
  223. return
  224. }
  225. }