lexer_test.go 7.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347
  1. /*
  2. * ECAL
  3. *
  4. * Copyright 2020 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the MIT
  7. * License, If a copy of the MIT License was not distributed with this
  8. * file, You can obtain one at https://opensource.org/licenses/MIT.
  9. */
  10. package parser
  11. import (
  12. "fmt"
  13. "testing"
  14. )
  15. func TestNextItem(t *testing.T) {
  16. l := &lexer{"Test", "1234", 0, 0, 0, 0, 0, make(chan LexToken)}
  17. r := l.next(1)
  18. if r != '1' {
  19. t.Errorf("Unexpected token: %q", r)
  20. return
  21. }
  22. if r := l.next(0); r != '1' {
  23. t.Errorf("Unexpected token: %q", r)
  24. return
  25. }
  26. if r := l.next(0); r != '2' {
  27. t.Errorf("Unexpected token: %q", r)
  28. return
  29. }
  30. if r := l.next(1); r != '3' {
  31. t.Errorf("Unexpected token: %q", r)
  32. return
  33. }
  34. if r := l.next(2); r != '4' {
  35. t.Errorf("Unexpected token: %q", r)
  36. return
  37. }
  38. if r := l.next(0); r != '3' {
  39. t.Errorf("Unexpected token: %q", r)
  40. return
  41. }
  42. if r := l.next(0); r != '4' {
  43. t.Errorf("Unexpected token: %q", r)
  44. return
  45. }
  46. if r := l.next(0); r != RuneEOF {
  47. t.Errorf("Unexpected token: %q", r)
  48. return
  49. }
  50. }
  51. func TestEquals(t *testing.T) {
  52. l := LexToList("mytest", "not\n test")
  53. if mt := l[0].Type(); mt != "MetaDataGeneral" {
  54. t.Error("Unexpected meta type:", mt)
  55. return
  56. }
  57. if ok, msg := l[0].Equals(l[1], false); ok || msg != `ID is different 54 vs 7
  58. Pos is different 0 vs 5
  59. Val is different not vs test
  60. Identifier is different false vs true
  61. Lline is different 1 vs 2
  62. Lpos is different 1 vs 2
  63. {
  64. "ID": 54,
  65. "Pos": 0,
  66. "Val": "not",
  67. "Identifier": false,
  68. "AllowEscapes": false,
  69. "Lsource": "mytest",
  70. "Lline": 1,
  71. "Lpos": 1
  72. }
  73. vs
  74. {
  75. "ID": 7,
  76. "Pos": 5,
  77. "Val": "test",
  78. "Identifier": true,
  79. "AllowEscapes": false,
  80. "Lsource": "mytest",
  81. "Lline": 2,
  82. "Lpos": 2
  83. }` {
  84. t.Error("Unexpected result:", msg)
  85. return
  86. }
  87. }
  88. func TestBasicTokenLexing(t *testing.T) {
  89. // Test empty string parsing
  90. if res := fmt.Sprint(LexToList("mytest", " \t ")); res != "[EOF]" {
  91. t.Error("Unexpected lexer result:\n ", res)
  92. return
  93. }
  94. // Test arithmetics
  95. input := `name := a + 1 and (ver+x!=1) * 5 > name2`
  96. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  97. `["name" := "a" + v:"1" <AND> ( "ver" + "x" != v:"1" ) * v:"5" > "name2" EOF]` {
  98. t.Error("Unexpected lexer result:\n ", res)
  99. return
  100. }
  101. input = `test := not a * 1.3 or (12 / aa) * 5 DiV 3 % 1 > trUe`
  102. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  103. `["test" := <NOT> "a" * v:"1.3" <OR> ( v:"12" / "aa" ) * v:"5" "DiV" v:"3" % v:"1" > <TRUE> EOF]` {
  104. t.Error("Unexpected lexer result:\n ", res)
  105. return
  106. }
  107. input = `-1.234560e+02+5+2.123 // 1`
  108. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  109. `[- v:"1.234560e+02" + v:"5" + v:"2.123" // v:"1" EOF]` {
  110. t.Error("Unexpected lexer result:\n ", res)
  111. return
  112. }
  113. // Test invalid identifier
  114. input = `5test`
  115. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  116. `[v:"5" "test" EOF]` {
  117. t.Error("Unexpected lexer result:\n ", res)
  118. return
  119. }
  120. input = `@test`
  121. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  122. `[Error: Cannot parse identifier '@test'. Identifies may only contain [a-zA-Z] and [a-zA-Z0-9] from the second character (Line 1, Pos 1) EOF]` {
  123. t.Error("Unexpected lexer result:\n ", res)
  124. return
  125. }
  126. }
  127. func TestAssignmentLexing(t *testing.T) {
  128. input := `name := a + 1`
  129. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  130. `["name" := "a" + v:"1" EOF]` {
  131. t.Error("Unexpected lexer result:", res)
  132. return
  133. }
  134. input = `name := a.a + a.b`
  135. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  136. `["name" := "a" . "a" + "a" . "b" EOF]` {
  137. t.Error("Unexpected lexer result:", res)
  138. return
  139. }
  140. input = `name:=a[1] + b["d"] + c[a]`
  141. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  142. `["name" := "a" [ v:"1" ] + "b" [ v:"d" ] + "c" [ "a" ] EOF]` {
  143. t.Error("Unexpected lexer result:", res)
  144. return
  145. }
  146. }
  147. func TestBlockLexing(t *testing.T) {
  148. input := `
  149. if a == 1 {
  150. print("xxx")
  151. } elif b > 2 {
  152. print("yyy")
  153. } else {
  154. print("zzz")
  155. }
  156. `
  157. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  158. `[<IF> "a" == v:"1" { "print" ( v:"xxx" ) } <ELIF> "b" > v:"2" { "print" ( v:"yyy" ) } <ELSE> { "print" ( v:"zzz" ) } EOF]` {
  159. t.Error("Unexpected lexer result:", res)
  160. return
  161. }
  162. input = `
  163. for a, b in enum(blist) {
  164. do(a)
  165. }
  166. `
  167. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  168. `[<FOR> "a" , "b" <IN> "enum" ( "blist" ) { "do" ( "a" ) } EOF]` {
  169. t.Error("Unexpected lexer result:", res)
  170. return
  171. }
  172. input = `
  173. for true {
  174. x := "1"
  175. break; continue
  176. }
  177. `
  178. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  179. `[<FOR> <TRUE> { "x" := v:"1" <BREAK> ; <CONTINUE> } EOF]` {
  180. t.Error("Unexpected lexer result:", res)
  181. return
  182. }
  183. }
  184. func TestStringLexing(t *testing.T) {
  185. // Test unclosed quotes
  186. input := `name "test bla`
  187. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading string value (unclosed quotes) (Line 1, Pos 6) EOF]` {
  188. t.Error("Unexpected lexer result:", res)
  189. return
  190. }
  191. input = `name "test" 'bla'`
  192. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" v:"test" v:"bla" EOF]` {
  193. t.Error("Unexpected lexer result:", res)
  194. return
  195. }
  196. input = `name "te
  197. st" 'bla'`
  198. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: invalid syntax while parsing string (Line 1, Pos 6)]` {
  199. t.Error("Unexpected lexer result:", res)
  200. return
  201. }
  202. input = `name r"te
  203. st" 'bla'`
  204. res := LexToList("mytest", input)
  205. if fmt.Sprint(res) != `["name" v:"te\n\tst" v:"bla" EOF]` {
  206. t.Error("Unexpected lexer result:", res)
  207. return
  208. }
  209. if res[1].AllowEscapes {
  210. t.Error("String value should not allow escapes")
  211. return
  212. }
  213. // Parsing with escape sequences
  214. input = `"test\n\ttest" '\nfoo\u0028bar' "test{foo}.5w3f"`
  215. res = LexToList("mytest", input)
  216. if fmt.Sprint(res) != `[v:"test\n\ttest" v:"\nfoo(bar" v:"test{foo}.5w3f" EOF]` {
  217. t.Error("Unexpected lexer result:", res)
  218. return
  219. }
  220. if !res[0].AllowEscapes {
  221. t.Error("String value should allow escapes")
  222. return
  223. }
  224. }
  225. func TestCommentLexing(t *testing.T) {
  226. input := `name /* foo
  227. bar
  228. x*/ 'b/* - */la' /*test*/`
  229. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" /* foo
  230. bar
  231. x */ v:"b/* - */la" /* test */ EOF]` {
  232. t.Error("Unexpected lexer result:", res)
  233. return
  234. }
  235. input = `name /* foo
  236. bar`
  237. if res := LexToList("mytest", input); fmt.Sprint(res) != `["name" Error: Unexpected end while reading comment (Line 1, Pos 8) EOF]` {
  238. t.Error("Unexpected lexer result:", res)
  239. return
  240. }
  241. input = `foo
  242. 1+ 2 # Some comment
  243. bar`
  244. if res := LexToList("mytest", input); fmt.Sprint(res) != `["foo" v:"1" + v:"2" # Some comment
  245. "bar" EOF]` {
  246. t.Error("Unexpected lexer result:", res)
  247. return
  248. }
  249. input = `1+ 2 # Some comment`
  250. if res := LexToList("mytest", input); fmt.Sprint(res) != `[v:"1" + v:"2" # Some comment EOF]` {
  251. t.Error("Unexpected lexer result:", res)
  252. return
  253. }
  254. input = `
  255. /*
  256. Conway's Game of Life
  257. A zero-player game that evolves based on its initial state.
  258. https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life
  259. */
  260. 1+ 2 # Some comment`
  261. if res := LexToList("mytest", input); fmt.Sprint(res) != `[/*
  262. Conway's Game of Life
  263. A zero-player game that evolves based on its initial state.
  264. https://en.wikipedia.org/wiki/Conway%27s_Game_of_Life
  265. */ v:"1" + v:"2" # Some comment EOF]` {
  266. t.Error("Unexpected lexer result:", res)
  267. return
  268. }
  269. }
  270. func TestSinkLexing(t *testing.T) {
  271. input := `sink "mysink"
  272. r"
  273. A comment describing the sink.
  274. "
  275. kindmatch [ foo.bar.* ],
  276. scopematch [ "data.read", "data.write" ],
  277. statematch { a : 1, b : NULL },
  278. priority 0,
  279. suppresses [ "myothersink" ]
  280. {
  281. a := 1
  282. }`
  283. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<SINK> v:"mysink" v:"\nA comment"... <KINDMATCH> `+
  284. `[ "foo" . "bar" . * ] , <SCOPEMATCH> [ v:"data.read" , v:"data.write" ] , <STATEMATCH> `+
  285. `{ "a" : v:"1" , "b" : <NULL> } , <PRIORITY> v:"0" , <SUPPRESSES> [ v:"myothersink" ] `+
  286. `{ "a" := v:"1" } EOF]` {
  287. t.Error("Unexpected lexer result:", res)
  288. return
  289. }
  290. }