lexer_test.go 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345
  1. /*
  2. * ECAL
  3. *
  4. * Copyright 2020 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the MIT
  7. * License, If a copy of the MIT License was not distributed with this
  8. * file, You can obtain one at https://opensource.org/licenses/MIT.
  9. */
  10. package parser
  11. import (
  12. "fmt"
  13. "testing"
  14. )
  15. func TestNextItem(t *testing.T) {
  16. l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, make(chan LexToken)}
  17. r := l.next(1)
  18. if r != '1' {
  19. t.Errorf("Unexpected token: %q", r)
  20. return
  21. }
  22. if r := l.next(0); r != '1' {
  23. t.Errorf("Unexpected token: %q", r)
  24. return
  25. }
  26. if r := l.next(0); r != '2' {
  27. t.Errorf("Unexpected token: %q", r)
  28. return
  29. }
  30. if r := l.next(1); r != '3' {
  31. t.Errorf("Unexpected token: %q", r)
  32. return
  33. }
  34. if r := l.next(2); r != '4' {
  35. t.Errorf("Unexpected token: %q", r)
  36. return
  37. }
  38. if r := l.next(0); r != '3' {
  39. t.Errorf("Unexpected token: %q", r)
  40. return
  41. }
  42. if r := l.next(0); r != '4' {
  43. t.Errorf("Unexpected token: %q", r)
  44. return
  45. }
  46. if r := l.next(0); r != RuneEOF {
  47. t.Errorf("Unexpected token: %q", r)
  48. return
  49. }
  50. }
  51. func TestEquals(t *testing.T) {
  52. l := LexToList("mytest", "not\n test")
  53. if mt := l[0].Type(); mt != "MetaDataGeneral" {
  54. t.Error("Unexpected meta type:", mt)
  55. return
  56. }
  57. if ok, msg := l[0].Equals(l[1], false); ok || msg != `ID is different 53 vs 7
  58. Pos is different 0 vs 5
  59. Val is different not vs test
  60. Identifier is different false vs true
  61. Lline is different 1 vs 2
  62. Lpos is different 1 vs 2
  63. {
  64. "ID": 53,
  65. "Pos": 0,
  66. "Val": "not",
  67. "Identifier": false,
  68. "AllowEscapes": false,
  69. "Lline": 1,
  70. "Lpos": 1
  71. }
  72. vs
  73. {
  74. "ID": 7,
  75. "Pos": 5,
  76. "Val": "test",
  77. "Identifier": true,
  78. "AllowEscapes": false,
  79. "Lline": 2,
  80. "Lpos": 2
  81. }` {
  82. t.Error("Unexpected result:", msg)
  83. return
  84. }
  85. }
  86. func TestBasicTokenLexing(t *testing.T) {
  87. // Test empty string parsing
  88. if res := fmt.Sprint(LexToList("mytest", " \t ")); res != "[EOF]" {
  89. t.Error("Unexpected lexer result:\n ", res)
  90. return
  91. }
  92. // Test arithmetics
  93. input := `name := a + 1 and (ver+x!=1) * 5 > name2`
  94. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  95. `["name" := "a" + v:"1" <AND> ( "ver" + "x" != v:"1" ) * v:"5" > "name2" EOF]` {
  96. t.Error("Unexpected lexer result:\n ", res)
  97. return
  98. }
  99. input = `test := not a * 1.3 or (12 / aa) * 5 DiV 3 % 1 > trUe`
  100. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  101. `["test" := <NOT> "a" * v:"1.3" <OR> ( v:"12" / "aa" ) * v:"5" "DiV" v:"3" % v:"1" > <TRUE> EOF]` {
  102. t.Error("Unexpected lexer result:\n ", res)
  103. return
  104. }
  105. input = `-1.234560e+02+5+2.123 // 1`
  106. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  107. `[- v:"1.234560e+02" + v:"5" + v:"2.123" // v:"1" EOF]` {
  108. t.Error("Unexpected lexer result:\n ", res)
  109. return
  110. }
  111. // Test invalid identifier
  112. input = `5test`
  113. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  114. `[v:"5" "test" EOF]` {
  115. t.Error("Unexpected lexer result:\n ", res)
  116. return
  117. }
  118. input = `@test`
  119. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  120. `[Error: Cannot parse identifier '@test'. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character (Line 1, Pos 1) EOF]` {
  121. t.Error("Unexpected lexer result:\n ", res)
  122. return
  123. }
  124. }
  125. func TestAssignmentLexing(t *testing.T) {
  126. input := `name := a + 1`
  127. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  128. `["name" := "a" + v:"1" EOF]` {
  129. t.Error("Unexpected lexer result:", res)
  130. return
  131. }
  132. input = `name := a.a + a.b`
  133. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  134. `["name" := "a" . "a" + "a" . "b" EOF]` {
  135. t.Error("Unexpected lexer result:", res)
  136. return
  137. }
  138. input = `name:=a[1] + b["d"] + c[a]`
  139. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  140. `["name" := "a" [ v:"1" ] + "b" [ v:"d" ] + "c" [ "a" ] EOF]` {
  141. t.Error("Unexpected lexer result:", res)
  142. return
  143. }
  144. }
  145. func TestBlockLexing(t *testing.T) {
  146. input := `
  147. if a == 1 {
  148. print("xxx")
  149. } elif b > 2 {
  150. print("yyy")
  151. } else {
  152. print("zzz")
  153. }
  154. `
  155. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  156. `[<IF> "a" == v:"1" { "print" ( v:"xxx" ) } <ELIF> "b" > v:"2" { "print" ( v:"yyy" ) } <ELSE> { "print" ( v:"zzz" ) } EOF]` {
  157. t.Error("Unexpected lexer result:", res)
  158. return
  159. }
  160. input = `
  161. for a, b in enum(blist) {
  162. do(a)
  163. }
  164. `
  165. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  166. `[<FOR> "a" , "b" <IN> "enum" ( "blist" ) { "do" ( "a" ) } EOF]` {
  167. t.Error("Unexpected lexer result:", res)
  168. return
  169. }
  170. input = `
  171. for true {
  172. x := "1"
  173. break; continue
  174. }
  175. `
  176. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  177. `[<FOR> <TRUE> { "x" := v:"1" <BREAK> ; <CONTINUE> } EOF]` {
  178. t.Error("Unexpected lexer result:", res)
  179. return
  180. }
  181. }
  182. func TestStringLexing(t *testing.T) {
  183. // Test unclosed quotes
  184. input := `name "test bla`
  185. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading string value (unclosed quotes) (Line 1, Pos 6) EOF]` {
  186. t.Error("Unexpected lexer result:", res)
  187. return
  188. }
  189. input = `name "test" 'bla'`
  190. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" v:"test" v:"bla" EOF]` {
  191. t.Error("Unexpected lexer result:", res)
  192. return
  193. }
  194. input = `name "te
  195. st" 'bla'`
  196. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: invalid syntax while parsing string (Line 1, Pos 6)]` {
  197. t.Error("Unexpected lexer result:", res)
  198. return
  199. }
  200. input = `name r"te
  201. st" 'bla'`
  202. res := LexToList("mytest", input)
  203. if fmt.Sprint(res) != `["name" v:"te\n\tst" v:"bla" EOF]` {
  204. t.Error("Unexpected lexer result:", res)
  205. return
  206. }
  207. if res[1].AllowEscapes {
  208. t.Error("String value should not allow escapes")
  209. return
  210. }
  211. // Parsing with escape sequences
  212. input = `"test\n\ttest" '\nfoo\u0028bar' "test{foo}.5w3f"`
  213. res = LexToList("mytest", input)
  214. if fmt.Sprint(res) != `[v:"test\n\ttest" v:"\nfoo(bar" v:"test{foo}.5w3f" EOF]` {
  215. t.Error("Unexpected lexer result:", res)
  216. return
  217. }
  218. if !res[0].AllowEscapes {
  219. t.Error("String value should allow escapes")
  220. return
  221. }
  222. }
  223. func TestCommentLexing(t *testing.T) {
  224. input := `name /* foo
  225. bar
  226. x*/ 'b/* - */la' /*test*/`
  227. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" /* foo
  228. bar
  229. x */ v:"b/* - */la" /* test */ EOF]` {
  230. t.Error("Unexpected lexer result:", res)
  231. return
  232. }
  233. input = `name /* foo
  234. bar`
  235. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading comment (Line 1, Pos 8) EOF]` {
  236. t.Error("Unexpected lexer result:", res)
  237. return
  238. }
  239. input = `foo
  240. 1+ 2 # Some comment
  241. bar`
  242. if res := LexToList("mytest", input); fmt.Sprint(res) != `["foo" v:"1" + v:"2" # Some comment
  243. "bar" EOF]` {
  244. t.Error("Unexpected lexer result:", res)
  245. return
  246. }
  247. input = `1+ 2 # Some comment`
  248. if res := LexToList("mytest", input); fmt.Sprint(res) != `[v:"1" + v:"2" # Some comment EOF]` {
  249. t.Error("Unexpected lexer result:", res)
  250. return
  251. }
  252. input = `
  253. /*
  254. Conway's Game of Life
  255. A zero-player game that evolves based on its initial state.
  256. https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life
  257. */
  258. 1+ 2 # Some comment`
  259. if res := LexToList("mytest", input); fmt.Sprint(res) != `[/*
  260. Conway's Game of Life
  261. A zero-player game that evolves based on its initial state.
  262. https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life
  263. */ v:"1" + v:"2" # Some comment EOF]` {
  264. t.Error("Unexpected lexer result:", res)
  265. return
  266. }
  267. }
  268. func TestSinkLexing(t *testing.T) {
  269. input := `sink "mysink"
  270. r"
  271. A comment describing the sink.
  272. "
  273. kindmatch [ foo.bar.* ],
  274. scopematch [ "data.read", "data.write" ],
  275. statematch { a : 1, b : NULL },
  276. priority 0,
  277. suppresses [ "myothersink" ]
  278. {
  279. a := 1
  280. }`
  281. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<SINK> v:"mysink" v:"\nA comment"... <KINDMATCH> `+
  282. `[ "foo" . "bar" . * ] , <SCOPEMATCH> [ v:"data.read" , v:"data.write" ] , <STATEMATCH> `+
  283. `{ "a" : v:"1" , "b" : <NULL> } , <PRIORITY> v:"0" , <SUPPRESSES> [ v:"myothersink" ] `+
  284. `{ "a" := v:"1" } EOF]` {
  285. t.Error("Unexpected lexer result:", res)
  286. return
  287. }
  288. }