lexer_test.go 8.0 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. /*
  2. * ECAL
  3. *
  4. * Copyright 2020 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the MIT
  7. * License, If a copy of the MIT License was not distributed with this
  8. * file, You can obtain one at https://opensource.org/licenses/MIT.
  9. */
  10. package parser
  11. import (
  12. "fmt"
  13. "testing"
  14. )
  15. func TestNextItem(t *testing.T) {
  16. l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, 0, make(chan LexToken)}
  17. r := l.next(1)
  18. if r != '1' {
  19. t.Errorf("Unexpected token: %q", r)
  20. return
  21. }
  22. if r := l.next(0); r != '1' {
  23. t.Errorf("Unexpected token: %q", r)
  24. return
  25. }
  26. if r := l.next(0); r != '2' {
  27. t.Errorf("Unexpected token: %q", r)
  28. return
  29. }
  30. if r := l.next(1); r != '3' {
  31. t.Errorf("Unexpected token: %q", r)
  32. return
  33. }
  34. if r := l.next(2); r != '4' {
  35. t.Errorf("Unexpected token: %q", r)
  36. return
  37. }
  38. if r := l.next(0); r != '3' {
  39. t.Errorf("Unexpected token: %q", r)
  40. return
  41. }
  42. if r := l.next(0); r != '4' {
  43. t.Errorf("Unexpected token: %q", r)
  44. return
  45. }
  46. if r := l.next(0); r != RuneEOF {
  47. t.Errorf("Unexpected token: %q", r)
  48. return
  49. }
  50. }
  51. func TestEquals(t *testing.T) {
  52. l := LexToList("mytest", "not\n test")
  53. if mt := l[0].Type(); mt != "MetaDataGeneral" {
  54. t.Error("Unexpected meta type:", mt)
  55. return
  56. }
  57. if ok, msg := l[0].Equals(l[1], false); ok || msg != `ID is different 54 vs 7
  58. Pos is different 0 vs 5
  59. Val is different not vs test
  60. Identifier is different false vs true
  61. Lline is different 1 vs 2
  62. Lpos is different 1 vs 2
  63. {
  64. "ID": 54,
  65. "Pos": 0,
  66. "Val": "not",
  67. "Identifier": false,
  68. "AllowEscapes": false,
  69. "PrefixNewlines": 0,
  70. "Lsource": "mytest",
  71. "Lline": 1,
  72. "Lpos": 1
  73. }
  74. vs
  75. {
  76. "ID": 7,
  77. "Pos": 5,
  78. "Val": "test",
  79. "Identifier": true,
  80. "AllowEscapes": false,
  81. "PrefixNewlines": 1,
  82. "Lsource": "mytest",
  83. "Lline": 2,
  84. "Lpos": 2
  85. }` {
  86. t.Error("Unexpected result:", msg)
  87. return
  88. }
  89. }
  90. func TestBasicTokenLexing(t *testing.T) {
  91. // Test empty string parsing
  92. if res := fmt.Sprint(LexToList("mytest", " \t ")); res != "[EOF]" {
  93. t.Error("Unexpected lexer result:\n ", res)
  94. return
  95. }
  96. // Test arithmetics
  97. input := `name := a + 1 and (ver+x!=1) * 5 > name2`
  98. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  99. `["name" := "a" + v:"1" <AND> ( "ver" + "x" != v:"1" ) * v:"5" > "name2" EOF]` {
  100. t.Error("Unexpected lexer result:\n ", res)
  101. return
  102. }
  103. input = `test := not a * 1.3 or (12 / aa) * 5 DiV 3 % 1 > trUe`
  104. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  105. `["test" := <NOT> "a" * v:"1.3" <OR> ( v:"12" / "aa" ) * v:"5" "DiV" v:"3" % v:"1" > <TRUE> EOF]` {
  106. t.Error("Unexpected lexer result:\n ", res)
  107. return
  108. }
  109. input = `-1.234560e+02+5+2.123 // 1`
  110. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  111. `[- v:"1.234560e+02" + v:"5" + v:"2.123" // v:"1" EOF]` {
  112. t.Error("Unexpected lexer result:\n ", res)
  113. return
  114. }
  115. // Test invalid identifier
  116. input = `5test`
  117. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  118. `[v:"5" "test" EOF]` {
  119. t.Error("Unexpected lexer result:\n ", res)
  120. return
  121. }
  122. input = `@test`
  123. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  124. `[Error: Cannot parse identifier '@test'. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character (Line 1, Pos 1) EOF]` {
  125. t.Error("Unexpected lexer result:\n ", res)
  126. return
  127. }
  128. }
  129. func TestAssignmentLexing(t *testing.T) {
  130. input := `name := a + 1`
  131. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  132. `["name" := "a" + v:"1" EOF]` {
  133. t.Error("Unexpected lexer result:", res)
  134. return
  135. }
  136. input = `name := a.a + a.b`
  137. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  138. `["name" := "a" . "a" + "a" . "b" EOF]` {
  139. t.Error("Unexpected lexer result:", res)
  140. return
  141. }
  142. input = `name:=a[1] + b["d"] + c[a]`
  143. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  144. `["name" := "a" [ v:"1" ] + "b" [ v:"d" ] + "c" [ "a" ] EOF]` {
  145. t.Error("Unexpected lexer result:", res)
  146. return
  147. }
  148. }
  149. func TestBlockLexing(t *testing.T) {
  150. input := `
  151. if a == 1 {
  152. print("xxx")
  153. } elif b > 2 {
  154. print("yyy")
  155. } else {
  156. print("zzz")
  157. }
  158. `
  159. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  160. `[<IF> "a" == v:"1" { "print" ( v:"xxx" ) } <ELIF> "b" > v:"2" { "print" ( v:"yyy" ) } <ELSE> { "print" ( v:"zzz" ) } EOF]` {
  161. t.Error("Unexpected lexer result:", res)
  162. return
  163. }
  164. input = `
  165. for a, b in enum(blist) {
  166. do(a)
  167. }
  168. `
  169. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  170. `[<FOR> "a" , "b" <IN> "enum" ( "blist" ) { "do" ( "a" ) } EOF]` {
  171. t.Error("Unexpected lexer result:", res)
  172. return
  173. }
  174. input = `
  175. for true {
  176. x := "1"
  177. break; continue
  178. }
  179. `
  180. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  181. `[<FOR> <TRUE> { "x" := v:"1" <BREAK> ; <CONTINUE> } EOF]` {
  182. t.Error("Unexpected lexer result:", res)
  183. return
  184. }
  185. }
  186. func TestStringLexing(t *testing.T) {
  187. // Test unclosed quotes
  188. input := `name "test bla`
  189. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading string value (unclosed quotes) (Line 1, Pos 6) EOF]` {
  190. t.Error("Unexpected lexer result:", res)
  191. return
  192. }
  193. input = `name "test" 'bla'`
  194. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" v:"test" v:"bla" EOF]` {
  195. t.Error("Unexpected lexer result:", res)
  196. return
  197. }
  198. input = `name "te
  199. st" 'bla'`
  200. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: invalid syntax while parsing string (Line 1, Pos 6)]` {
  201. t.Error("Unexpected lexer result:", res)
  202. return
  203. }
  204. input = `name r"te
  205. st" 'bla'`
  206. res := LexToList("mytest", input)
  207. if fmt.Sprint(res) != `["name" v:"te\n\tst" v:"bla" EOF]` {
  208. t.Error("Unexpected lexer result:", res)
  209. return
  210. }
  211. if res[1].AllowEscapes {
  212. t.Error("String value should not allow escapes")
  213. return
  214. }
  215. // Parsing with escape sequences
  216. input = `"test\n\ttest" '\nfoo\u0028bar' "test{foo}.5w3f"`
  217. res = LexToList("mytest", input)
  218. if fmt.Sprint(res) != `[v:"test\n\ttest" v:"\nfoo(bar" v:"test{foo}.5w3f" EOF]` {
  219. t.Error("Unexpected lexer result:", res)
  220. return
  221. }
  222. if !res[0].AllowEscapes {
  223. t.Error("String value should allow escapes")
  224. return
  225. }
  226. }
  227. func TestCommentLexing(t *testing.T) {
  228. input := `name /* foo
  229. bar
  230. x*/ 'b/* - */la' /*test*/`
  231. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" /* foo
  232. bar
  233. x */ v:"b/* - */la" /* test */ EOF]` {
  234. t.Error("Unexpected lexer result:", res)
  235. return
  236. }
  237. input = `name /* foo
  238. bar`
  239. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading comment (Line 1, Pos 8) EOF]` {
  240. t.Error("Unexpected lexer result:", res)
  241. return
  242. }
  243. input = `foo
  244. 1+ 2 # Some comment
  245. bar`
  246. if res := LexToList("mytest", input); fmt.Sprint(res) != `["foo" v:"1" + v:"2" # Some comment
  247. "bar" EOF]` {
  248. t.Error("Unexpected lexer result:", res)
  249. return
  250. }
  251. input = `1+ 2 # Some comment`
  252. if res := LexToList("mytest", input); fmt.Sprint(res) != `[v:"1" + v:"2" # Some comment EOF]` {
  253. t.Error("Unexpected lexer result:", res)
  254. return
  255. }
  256. input = `
  257. /*
  258. Conway's Game of Life
  259. A zero-player game that evolves based on its initial state.
  260. https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life
  261. */
  262. 1+ 2 # Some comment`
  263. if res := LexToList("mytest", input); fmt.Sprint(res) != `[/*
  264. Conway's Game of Life
  265. A zero-player game that evolves based on its initial state.
  266. https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life
  267. */ v:"1" + v:"2" # Some comment EOF]` {
  268. t.Error("Unexpected lexer result:", res)
  269. return
  270. }
  271. }
  272. func TestSinkLexing(t *testing.T) {
  273. input := `sink "mysink"
  274. r"
  275. A comment describing the sink.
  276. "
  277. kindmatch [ foo.bar.* ],
  278. scopematch [ "data.read", "data.write" ],
  279. statematch { a : 1, b : NULL },
  280. priority 0,
  281. suppresses [ "myothersink" ]
  282. {
  283. a := 1
  284. }`
  285. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<SINK> v:"mysink" v:"\nA comment"... <KINDMATCH> `+
  286. `[ "foo" . "bar" . * ] , <SCOPEMATCH> [ v:"data.read" , v:"data.write" ] , <STATEMATCH> `+
  287. `{ "a" : v:"1" , "b" : <NULL> } , <PRIORITY> v:"0" , <SUPPRESSES> [ v:"myothersink" ] `+
  288. `{ "a" := v:"1" } EOF]` {
  289. t.Error("Unexpected lexer result:", res)
  290. return
  291. }
  292. }