lexer_test.go 7.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309
  1. /*
  2. * Public Domain Software
  3. *
  4. * I (Matthias Ladkau) am the author of the source code in this file.
  5. * I have placed the source code in this file in the public domain.
  6. *
  7. * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
  8. */
  9. package parser
  10. import (
  11. "fmt"
  12. "testing"
  13. )
  14. func TestNextItem(t *testing.T) {
  15. l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, make(chan LexToken)}
  16. r := l.next(1)
  17. if r != '1' {
  18. t.Errorf("Unexpected token: %q", r)
  19. return
  20. }
  21. if r := l.next(0); r != '1' {
  22. t.Errorf("Unexpected token: %q", r)
  23. return
  24. }
  25. if r := l.next(0); r != '2' {
  26. t.Errorf("Unexpected token: %q", r)
  27. return
  28. }
  29. if r := l.next(1); r != '3' {
  30. t.Errorf("Unexpected token: %q", r)
  31. return
  32. }
  33. if r := l.next(2); r != '4' {
  34. t.Errorf("Unexpected token: %q", r)
  35. return
  36. }
  37. if r := l.next(0); r != '3' {
  38. t.Errorf("Unexpected token: %q", r)
  39. return
  40. }
  41. if r := l.next(0); r != '4' {
  42. t.Errorf("Unexpected token: %q", r)
  43. return
  44. }
  45. if r := l.next(0); r != RuneEOF {
  46. t.Errorf("Unexpected token: %q", r)
  47. return
  48. }
  49. }
  50. func TestEquals(t *testing.T) {
  51. l := LexToList("mytest", "not\n test")
  52. if mt := l[0].Type(); mt != "MetaDataGeneral" {
  53. t.Error("Unexpected meta type:", mt)
  54. return
  55. }
  56. if ok, msg := l[0].Equals(l[1], false); ok || msg != `ID is different 47 vs 7
  57. Pos is different 0 vs 5
  58. Val is different not vs test
  59. Identifier is different false vs true
  60. Lline is different 1 vs 2
  61. Lpos is different 1 vs 2
  62. {
  63. "ID": 47,
  64. "Pos": 0,
  65. "Val": "not",
  66. "Identifier": false,
  67. "Lline": 1,
  68. "Lpos": 1
  69. }
  70. vs
  71. {
  72. "ID": 7,
  73. "Pos": 5,
  74. "Val": "test",
  75. "Identifier": true,
  76. "Lline": 2,
  77. "Lpos": 2
  78. }` {
  79. t.Error("Unexpected result:", msg)
  80. return
  81. }
  82. }
  83. func TestBasicTokenLexing(t *testing.T) {
  84. // Test empty string parsing
  85. if res := fmt.Sprint(LexToList("mytest", " \t ")); res != "[EOF]" {
  86. t.Error("Unexpected lexer result:\n ", res)
  87. return
  88. }
  89. // Test arithmetics
  90. input := `name := a + 1 and (ver+x!=1) * 5 > name2`
  91. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  92. `["name" := "a" + v:"1" <AND> ( "ver" + "x" != v:"1" ) * v:"5" > "name2" EOF]` {
  93. t.Error("Unexpected lexer result:\n ", res)
  94. return
  95. }
  96. input = `test := not a * 1.3 or (12 / aa) * 5 DIV 3 % 1 > true`
  97. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  98. `["test" := <NOT> "a" * v:"1.3" <OR> ( v:"12" / "aa" ) * v:"5" "div" v:"3" % v:"1" > <TRUE> EOF]` {
  99. t.Error("Unexpected lexer result:\n ", res)
  100. return
  101. }
  102. input = `-1.234560e+02+5+2.123 // 1`
  103. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  104. `[- v:"1.234560e+02" + v:"5" + v:"2.123" // v:"1" EOF]` {
  105. t.Error("Unexpected lexer result:\n ", res)
  106. return
  107. }
  108. // Test invalid identifier
  109. input = `5test`
  110. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  111. `[v:"5" "test" EOF]` {
  112. t.Error("Unexpected lexer result:\n ", res)
  113. return
  114. }
  115. input = `@test`
  116. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  117. `[Error: Cannot parse identifier '@test'. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character (Line 1, Pos 1) EOF]` {
  118. t.Error("Unexpected lexer result:\n ", res)
  119. return
  120. }
  121. }
  122. func TestAssignmentLexing(t *testing.T) {
  123. input := `name := a + 1`
  124. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  125. `["name" := "a" + v:"1" EOF]` {
  126. t.Error("Unexpected lexer result:", res)
  127. return
  128. }
  129. input = `name := a.a + a.b`
  130. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  131. `["name" := "a" . "a" + "a" . "b" EOF]` {
  132. t.Error("Unexpected lexer result:", res)
  133. return
  134. }
  135. input = `name:=a[1] + b["d"] + c[a]`
  136. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  137. `["name" := "a" [ v:"1" ] + "b" [ "d" ] + "c" [ "a" ] EOF]` {
  138. t.Error("Unexpected lexer result:", res)
  139. return
  140. }
  141. }
  142. func TestBlockLexing(t *testing.T) {
  143. input := `
  144. if a == 1 {
  145. print("xxx")
  146. } elif b > 2 {
  147. print("yyy")
  148. } else {
  149. print("zzz")
  150. }
  151. `
  152. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  153. `[<IF> "a" == v:"1" { "print" ( "xxx" ) } <ELIF> "b" > v:"2" { "print" ( "yyy" ) } <ELSE> { "print" ( "zzz" ) } EOF]` {
  154. t.Error("Unexpected lexer result:", res)
  155. return
  156. }
  157. input = `
  158. for a, b in enum(blist) {
  159. do(a)
  160. }
  161. `
  162. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  163. `[<FOR> "a" , "b" <IN> "enum" ( "blist" ) { "do" ( "a" ) } EOF]` {
  164. t.Error("Unexpected lexer result:", res)
  165. return
  166. }
  167. input = `
  168. for true {
  169. x := "1"
  170. break; continue
  171. }
  172. `
  173. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  174. `[<FOR> <TRUE> { "x" := "1" <BREAK> ; <CONTINUE> } EOF]` {
  175. t.Error("Unexpected lexer result:", res)
  176. return
  177. }
  178. }
  179. func TestStringLexing(t *testing.T) {
  180. // Test unclosed quotes
  181. input := `name "test bla`
  182. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading string value (unclosed quotes) (Line 1, Pos 6) EOF]` {
  183. t.Error("Unexpected lexer result:", res)
  184. return
  185. }
  186. input = `name "test" 'bla'`
  187. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" "test" "bla" EOF]` {
  188. t.Error("Unexpected lexer result:", res)
  189. return
  190. }
  191. input = `name "te
  192. st" 'bla'`
  193. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: invalid syntax while parsing string (Line 1, Pos 6)]` {
  194. t.Error("Unexpected lexer result:", res)
  195. return
  196. }
  197. input = `name r"te
  198. st" 'bla'`
  199. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" "te\n\tst" "bla" EOF]` {
  200. t.Error("Unexpected lexer result:", res)
  201. return
  202. }
  203. // Parsing with escape sequences
  204. input = `"test\n\ttest" '\nfoo\u0028bar' "test{foo}.5w3f"`
  205. if res := LexToList("mytest", input); fmt.Sprint(res) != `["test\n\ttest" "\nfoo(bar" "test{foo}.5w3f" EOF]` {
  206. t.Error("Unexpected lexer result:", res)
  207. return
  208. }
  209. }
  210. func TestCommentLexing(t *testing.T) {
  211. input := `name /* foo
  212. bar
  213. x*/ 'b/* - */la' /*test*/`
  214. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" /* foo
  215. bar
  216. x */ "b/* - */la" /* test */ EOF]` {
  217. t.Error("Unexpected lexer result:", res)
  218. return
  219. }
  220. input = `name /* foo
  221. bar`
  222. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading comment (Line 1, Pos 8) EOF]` {
  223. t.Error("Unexpected lexer result:", res)
  224. return
  225. }
  226. input = `foo
  227. 1+ 2 # Some comment
  228. bar`
  229. if res := LexToList("mytest", input); fmt.Sprint(res) != `["foo" v:"1" + v:"2" # Some comment
  230. "bar" EOF]` {
  231. t.Error("Unexpected lexer result:", res)
  232. return
  233. }
  234. input = `1+ 2 # Some comment`
  235. if res := LexToList("mytest", input); fmt.Sprint(res) != `[v:"1" + v:"2" # Some comment EOF]` {
  236. t.Error("Unexpected lexer result:", res)
  237. return
  238. }
  239. }
  240. func TestSinkLexing(t *testing.T) {
  241. input := `sink "mysink"
  242. r"
  243. A comment describing the sink.
  244. "
  245. kindmatch [ foo.bar.* ],
  246. scopematch [ "data.read", "data.write" ],
  247. statematch { a : 1, b : NULL },
  248. priority 0,
  249. suppresses [ "myothersink" ]
  250. {
  251. a := 1
  252. }`
  253. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<SINK> "mysink" "\nA comment"... <KINDMATCH> `+
  254. `[ "foo" . "bar" . * ] , <SCOPEMATCH> [ "data.read" , "data.write" ] , <STATEMATCH> `+
  255. `{ "a" : v:"1" , "b" : <NULL> } , <PRIORITY> v:"0" , <SUPPRESSES> [ "myothersink" ] `+
  256. `{ "a" := v:"1" } EOF]` {
  257. t.Error("Unexpected lexer result:", res)
  258. return
  259. }
  260. }