lexer_test.go 9.1 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package parser
  11. import (
  12. "fmt"
  13. "testing"
  14. )
  15. const complexTextQuery = `# This is a comment
  16. LOOKUP Song "a","b","c", "blaД"
  17. FROM GROUP test
  18. WHERE r'ДTest AttrblaД' = "blaД" and a < b and a = TRAVERS or not a--1 > b or c contains d and e = 10 - -1 * 10 + 5 or -b
  19. TRAVERSE Song:PerformedSong:Author:Author WHERE f = 6 # This is a comment
  20. TRAVERSE Author:PerformedAlbum:Album:Album
  21. END
  22. TRAVERSE :PerformedAlbum::Album
  23. END
  24. TRAVERSE :::
  25. END
  26. TRAVERSE Author:PerformedAlbum:Album:Album
  27. TRAVERSE Author:PerformedAlbum:Album:Album WHERE r:a beginsWith "jj" and b endsWith "kk"
  28. END
  29. END
  30. END
  31. TRAVERSE Data:StoredData:File:File WHERE a < b and 7 = (6 + 5) and @count("File:File:StoredData:Data") > 1
  32. END
  33. TRAVERSE Data:StoredData:File:File WHERE @someFunc(1,2,3,"Data",@someMore()) = -1
  34. END
  35. PRIMARY Author
  36. SHOW
  37. Song:title AS r'Title (mytitle)' FORMAT text,
  38. r'Song!2:t title' AS "Title test" FORMAT text:bla_bla_blub,
  39. Song:title AS Title FORMAT text:bla_bla_blub:dudududu,
  40. !4:kind
  41. WITH ORDERING(ASCENDING dd,DESCENDING dsd), FILTERING(ISNOTNULL ss,UNIQUE aa)
  42. `
  43. func TestComplexLexing(t *testing.T) {
  44. if res := LexToList("mytest", complexTextQuery); fmt.Sprint(res) !=
  45. `[<LOOKUP> "Song" "a" , "b" , "c" , "blaД" <FROM> <GROUP> "test" <WHERE> `+
  46. `"ДTest Attr"... = "blaД" <AND> "a" < "b" <AND> "a" = "TRAVERS" <OR> <NOT> `+
  47. `"a" - - "1" > "b" <OR> "c" <CONTAINS> "d" <AND> "e" = "10" - - "1" * "10" `+
  48. `+ "5" <OR> - "b" <TRAVERSE> "Song:Perfo"... <WHERE> "f" = "6" <TRAVERSE> `+
  49. `"Author:Per"... <END> <TRAVERSE> ":Performed"... <END> <TRAVERSE> ":::" `+
  50. `<END> <TRAVERSE> "Author:Per"... <TRAVERSE> "Author:Per"... <WHERE> "r:a" `+
  51. `<BEGINSWITH> "jj" <AND> "b" <ENDSWITH> "kk" <END> <END> <END> <TRAVERSE> `+
  52. `"Data:Store"... <WHERE> "a" < "b" <AND> "7" = ( "6" + "5" ) <AND> @ `+
  53. `"count" ( "File:File:"... ) > "1" <END> <TRAVERSE> "Data:Store"... `+
  54. `<WHERE> @ "someFunc" ( "1" , "2" , "3" , "Data" , @ "someMore" ( ) ) `+
  55. `= - "1" <END> <PRIMARY> "Author" <SHOW> "Song:title" <AS> "Title `+
  56. `(myt"... <FORMAT> "text" , "Song!2:t t"... <AS> "Title test" `+
  57. `<FORMAT> "text:bla_b"... , "Song:title" <AS> "Title" <FORMAT> `+
  58. `"text:bla_b"... , "!4:kind" <WITH> <ORDERING> ( <ASCENDING> "dd" , `+
  59. `<DESCENDING> "dsd" ) , <FILTERING> ( <ISNOTNULL> `+
  60. `"ss" , <UNIQUE> "aa" ) EOF]` {
  61. t.Error("Unexpected lexer result:", res)
  62. return
  63. }
  64. }
  65. func TestSimpleLexing(t *testing.T) {
  66. // Test empty string parsing
  67. if res := fmt.Sprint(LexToList("mytest", " \t ")); res != "[EOF]" {
  68. t.Error("Unexpected lexer result:", res)
  69. return
  70. }
  71. // Test invalid node kind
  72. input := " \n gEt \n my@node where xxx"
  73. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  74. "[<GET> Error: Invalid node kind 'my@node' - can only contain [a-zA-Z0-9_] (Line 3, Pos 2)]" {
  75. t.Error("Unexpected lexer result:", res)
  76. return
  77. }
  78. // Test valid node kind
  79. input = "GET mynode"
  80. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<GET> "mynode" EOF]` {
  81. t.Error("Unexpected lexer result:", res)
  82. return
  83. }
  84. // Test unquoted value parsing
  85. input = `GET mynode WHERE name = "myname:x"`
  86. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<GET> "mynode" <WHERE> "name" = "myname:x" EOF]` {
  87. t.Error("Unexpected lexer result:", res)
  88. return
  89. }
  90. // Test arithmetics
  91. input = `GET mynode WHERE name = a + 1 and (ver+x) * 5 > name2`
  92. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  93. `[<GET> "mynode" <WHERE> "name" = "a" + "1" <AND> ( "ver" + "x" ) * "5" > "name2" EOF]` {
  94. t.Error("Unexpected lexer result:", res)
  95. return
  96. }
  97. input = `GET mynode WHERE test = a * 1.3 and (12 / 55aa) * 5 DIV 3 % 1 > true`
  98. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  99. `[<GET> "mynode" <WHERE> "test" = "a" * "1.3" <AND> ( "12" / "55aa" ) * "5" "DIV" "3" % "1" > <TRUE> EOF]` {
  100. t.Error("Unexpected lexer result:", res)
  101. return
  102. }
  103. // Test comments
  104. input = `GET mynode # WHERE testcomment = a * 1.3
  105. WHERE a = b
  106. #end`
  107. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  108. `[<GET> "mynode" <WHERE> "a" = "b" EOF]` {
  109. t.Error("Unexpected lexer result:", res)
  110. return
  111. }
  112. // Test traversal
  113. input = `GET mynode WHERE Author = rabatt TRAVERSE Song:PerformedSong:Author:Author WHERE Author = 6 # This is a comment
  114. END`
  115. if res := LexToList("mytest", input); fmt.Sprint(res) !=
  116. `[<GET> "mynode" <WHERE> "Author" = "rabatt" <TRAVERSE> "Song:Perfo"... <WHERE> "Author" = "6" <END> EOF]` {
  117. t.Error("Unexpected lexer result:", res)
  118. return
  119. }
  120. }
  121. func TestValueParsing(t *testing.T) {
  122. // First word recognition
  123. if res := FirstWord(" aBBa test"); res != "aBBa" {
  124. t.Error("Unexpected first word:", res)
  125. return
  126. }
  127. if res := FirstWord("test"); res != "test" {
  128. t.Error("Unexpected first word:", res)
  129. return
  130. }
  131. if res := FirstWord(" test"); res != "test" {
  132. t.Error("Unexpected first word:", res)
  133. return
  134. }
  135. if res := FirstWord(" \n "); res != "" {
  136. t.Error("Unexpected first word:", res)
  137. return
  138. }
  139. if res := FirstWord(""); res != "" {
  140. t.Error("Unexpected first word:", res)
  141. return
  142. }
  143. // Test normal quoted case
  144. input := `WHERE "name"`
  145. if res := LexToList("mytest", input); res[1].Val != "name" {
  146. t.Error("Unexpected value:", res)
  147. return
  148. }
  149. // Test raw quoted
  150. input = `WHERE r'name'`
  151. if res := LexToList("mytest", input); res[1].Val != "name" {
  152. t.Error("Unexpected value:", res)
  153. return
  154. }
  155. // Test quoted with escape sequence
  156. input = `WHERE "\ntest"`
  157. if res := LexToList("mytest", input); res[1].Val != "\ntest" {
  158. t.Error("Unexpected value:", res)
  159. return
  160. }
  161. // Test raw input with spaces and uninterpreted escape sequence
  162. input = `WHERE r"name is not '\ntest'"`
  163. if res := LexToList("mytest", input); res[1].Val != "name is not '\\ntest'" {
  164. t.Error("Unexpected value:", res)
  165. return
  166. } else if s := fmt.Sprint(res[1]); s != `"name is no"...` {
  167. t.Error("Unexpected print result:", s)
  168. }
  169. // Test escape sequence error
  170. input = `WHERE "name\j"`
  171. if res := LexToList("mytest", input); res[1].Val != "invalid syntax while parsing escape sequences" || res[1].PosString() != "Line 1, Pos 7" {
  172. t.Error("Unexpected value:", res)
  173. return
  174. }
  175. // Test newline within string + error reporting
  176. input = ` WHERE r"name
  177. " WHERE bla`
  178. if res := LexToList("mytest", input); res[1].Val != "name\n " {
  179. t.Error("Unexpected value:", res)
  180. return
  181. }
  182. // Test correct line advancing
  183. input = ` WHERE r'name
  184. ' WHERE "bla
  185. "`
  186. if res := LexToList("mytest", input); res[3].Val != "invalid syntax while parsing escape sequences" || res[3].PosString() != "Line 2, Pos 10" {
  187. t.Error("Unexpected value:", res)
  188. return
  189. }
  190. // Test parse value error
  191. input = ` lookup x x`
  192. if res := LexToList("mytest", input); res[2].Val != "Value expected" || res[2].PosString() != "Line 1, Pos 11" {
  193. t.Error("Unexpected value:", res)
  194. return
  195. }
  196. input = ` lookup x "x`
  197. if res := LexToList("mytest", input); res[2].Val != "Unexpected end while reading value" || res[2].PosString() != "Line 1, Pos 11" {
  198. t.Error("Unexpected value:", res)
  199. return
  200. }
  201. input = `where aaaa!=aaa`
  202. if res := LexToList("mytest", input); fmt.Sprint(res) != `[<WHERE> "aaaa" != "aaa" EOF]` {
  203. t.Error("Unexpected value:", res)
  204. return
  205. }
  206. }
  207. func TestLexerInputControl(t *testing.T) {
  208. test := &lexer{"test", "test x\xe2\x8c\x98c", 0, 0, 0, 0, 0, -1, nil}
  209. if r := test.next(false); r != 't' {
  210. t.Error("Unexpected first rune:", r)
  211. return
  212. }
  213. if r := test.next(false); r != 'e' {
  214. t.Error("Unexpected first rune:", r)
  215. return
  216. }
  217. test.next(false)
  218. test.next(false)
  219. test.next(false)
  220. if r := test.next(false); r != 'x' {
  221. t.Error("Unexpected first rune:", r)
  222. return
  223. }
  224. if test.width != 1 {
  225. t.Error("Unexpected length of rune")
  226. return
  227. }
  228. // Test peeking
  229. if r := test.next(true); r != '\u2318' {
  230. t.Error("Unexpected first rune:", r)
  231. return
  232. }
  233. if r := test.next(false); r != '\u2318' {
  234. t.Error("Unexpected first rune:", r)
  235. return
  236. }
  237. if test.width != 3 {
  238. t.Error("Unexpected length of rune")
  239. return
  240. }
  241. if r := test.next(false); r != 'c' {
  242. t.Error("Unexpected first rune:", r)
  243. return
  244. }
  245. if test.width != 1 {
  246. t.Error("Unexpected length of rune")
  247. return
  248. }
  249. if test.next(false) != RuneEOF {
  250. t.Error("Unexpected last rune")
  251. return
  252. }
  253. if test.next(true) != RuneEOF {
  254. t.Error("Unexpected last rune")
  255. return
  256. }
  257. test.backup()
  258. if r := test.next(false); r != 'c' {
  259. t.Error("Unexpected first rune:", r)
  260. return
  261. }
  262. if test.width != 1 {
  263. t.Error("Unexpected length of rune")
  264. return
  265. }
  266. test.backup()
  267. if r := test.next(false); r != 'c' {
  268. t.Error("Unexpected first rune:", r)
  269. return
  270. }
  271. if test.width != 1 {
  272. t.Error("Unexpected length of rune")
  273. return
  274. }
  275. testBackupPanic(t, test)
  276. }
  277. func testBackupPanic(t *testing.T, l *lexer) {
  278. defer func() {
  279. if r := recover(); r == nil {
  280. t.Error("Doing a backup twice did not cause a panic.")
  281. }
  282. }()
  283. l.backup()
  284. l.backup()
  285. }