const.go 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. /*
  2. * ECAL
  3. *
  4. * Copyright 2020 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the MIT
  7. * License, If a copy of the MIT License was not distributed with this
  8. * file, You can obtain one at https://opensource.org/licenses/MIT.
  9. */
  10. /*
  11. Package parser contains a ECAL parser.
  12. Lexer for Source Text
  13. Lex() is a lexer function to convert a given search query into a list of tokens.
  14. Based on a talk by Rob Pike: Lexical Scanning in Go
  15. https://www.youtube.com/watch?v=HxaD_trXwRE
  16. The lexer's output is pushed into a channel which is consumed by the parser.
  17. This design enables the concurrent processing of the input text by lexer and
  18. parser.
  19. Parser
  20. Parse() is a parser which produces a parse tree from a given set of lexer tokens.
  21. Based on an article by Douglas Crockford: Top Down Operator Precedence
  22. http://crockford.com/javascript/tdop/tdop.html
  23. which is based on the ideas of Vaughan Pratt and his paper: Top Down Operator Precedence
  24. http://portal.acm.org/citation.cfm?id=512931
  25. https://tdop.github.io/
  26. ParseWithRuntime() parses a given input and decorates the resulting parse tree
  27. with runtime components which can be used to interpret the parsed query.
  28. */
  29. package parser
  30. /*
  31. LexTokenID represents a unique lexer token ID
  32. */
  33. type LexTokenID int
  34. /*
  35. Available meta data types
  36. */
  37. const (
  38. MetaDataPreComment = "MetaDataPreComment"
  39. MetaDataPostComment = "MetaDataPostComment"
  40. MetaDataGeneral = "MetaDataGeneral"
  41. )
  42. /*
  43. Available lexer token types
  44. */
  45. const (
  46. TokenError LexTokenID = iota // Lexing error token with a message as val
  47. TokenEOF // End-of-file token
  48. TokenANY // Unspecified token (used when building an AST from a Go map structure)
  49. TokenPRECOMMENT // Comment /* ... */
  50. TokenPOSTCOMMENT // Comment # ...
  51. // Value tokens
  52. TokenSTRING // String constant
  53. TokenNUMBER // Number constant
  54. TokenIDENTIFIER // Idendifier
  55. // Constructed tokens which are generated by the parser not the lexer
  56. TokenSTATEMENTS // A code block
  57. TokenFUNCCALL // A function call
  58. TokenCOMPACCESS // Access to a composition structure
  59. TokenLIST // List value
  60. TokenMAP // MAP value
  61. TokenPARAMS // Function parameters
  62. TokenGUARD // Conditional statements
  63. TOKENodeSYMBOLS // Used to separate symbols from other tokens in this list
  64. // Condition operators
  65. TokenGEQ
  66. TokenLEQ
  67. TokenNEQ
  68. TokenEQ
  69. TokenGT
  70. TokenLT
  71. // Grouping symbols
  72. TokenLPAREN
  73. TokenRPAREN
  74. TokenLBRACK
  75. TokenRBRACK
  76. TokenLBRACE
  77. TokenRBRACE
  78. // Separators
  79. TokenDOT
  80. TokenCOMMA
  81. TokenSEMICOLON
  82. // Grouping
  83. TokenCOLON
  84. TokenEQUAL
  85. // Arithmetic operators
  86. TokenPLUS
  87. TokenMINUS
  88. TokenTIMES
  89. TokenDIV
  90. TokenDIVINT
  91. TokenMODINT
  92. // Assignment statement
  93. TokenASSIGN
  94. TOKENodeKEYWORDS // Used to separate keywords from other tokens in this list
  95. // Import statement
  96. TokenIMPORT
  97. TokenAS
  98. // Sink definition
  99. TokenSINK
  100. TokenKINDMATCH
  101. TokenSCOPEMATCH
  102. TokenSTATEMATCH
  103. TokenPRIORITY
  104. TokenSUPPRESSES
  105. // Function definition
  106. TokenFUNC
  107. TokenRETURN
  108. // Boolean operators
  109. TokenAND
  110. TokenOR
  111. TokenNOT
  112. // Condition operators
  113. TokenLIKE
  114. TokenIN
  115. TokenHASPREFIX
  116. TokenHASSUFFIX
  117. TokenNOTIN
  118. // Constant terminals
  119. TokenFALSE
  120. TokenTRUE
  121. TokenNULL
  122. // Conditional statements
  123. TokenIF
  124. TokenELIF
  125. TokenELSE
  126. // Loop statements
  127. TokenFOR
  128. TokenBREAK
  129. TokenCONTINUE
  130. // Try block
  131. TokenTRY
  132. TokenEXCEPT
  133. TokenFINALLY
  134. TokenENDLIST
  135. )
  136. /*
  137. IsValidTokenID check if a given token ID is valid.
  138. */
  139. func IsValidTokenID(value int) bool {
  140. return value < int(TokenENDLIST)
  141. }
  142. /*
  143. Available parser AST node types
  144. */
  145. const (
  146. NodeEOF = "EOF"
  147. NodeSTRING = "string" // String constant
  148. NodeNUMBER = "number" // Number constant
  149. NodeIDENTIFIER = "identifier" // Idendifier
  150. // Constructed tokens
  151. NodeSTATEMENTS = "statements" // List of statements
  152. NodeFUNCCALL = "funccall" // Function call
  153. NodeCOMPACCESS = "compaccess" // Composition structure access
  154. NodeLIST = "list" // List value
  155. NodeMAP = "map" // Map value
  156. NodePARAMS = "params" // Function parameters
  157. NodeGUARD = "guard" // Guard expressions for conditional statements
  158. // Condition operators
  159. NodeGEQ = ">="
  160. NodeLEQ = "<="
  161. NodeNEQ = "!="
  162. NodeEQ = "=="
  163. NodeGT = ">"
  164. NodeLT = "<"
  165. // Separators
  166. NodeKVP = "kvp" // Key-value pair
  167. NodePRESET = "preset" // Preset value
  168. // Arithmetic operators
  169. NodePLUS = "plus"
  170. NodeMINUS = "minus"
  171. NodeTIMES = "times"
  172. NodeDIV = "div"
  173. NodeMODINT = "modint"
  174. NodeDIVINT = "divint"
  175. // Assignment statement
  176. NodeASSIGN = ":="
  177. // Import statement
  178. NodeIMPORT = "import"
  179. // Sink definition
  180. NodeSINK = "sink"
  181. NodeKINDMATCH = "kindmatch"
  182. NodeSCOPEMATCH = "scopematch"
  183. NodeSTATEMATCH = "statematch"
  184. NodePRIORITY = "priority"
  185. NodeSUPPRESSES = "suppresses"
  186. // Function definition
  187. NodeFUNC = "function"
  188. NodeRETURN = "return"
  189. // Boolean operators
  190. NodeAND = "and"
  191. NodeOR = "or"
  192. NodeNOT = "not"
  193. // Condition operators
  194. NodeLIKE = "like"
  195. NodeIN = "in"
  196. NodeHASPREFIX = "hasprefix"
  197. NodeHASSUFFIX = "hassuffix"
  198. NodeNOTIN = "notin"
  199. // Constant terminals
  200. NodeTRUE = "true"
  201. NodeFALSE = "false"
  202. NodeNULL = "null"
  203. // Conditional statements
  204. NodeIF = "if"
  205. // Loop statements
  206. NodeLOOP = "loop"
  207. NodeBREAK = "break"
  208. NodeCONTINUE = "continue"
  209. // Try block
  210. NodeTRY = "try"
  211. NodeEXCEPT = "except"
  212. NodeAS = "as"
  213. NodeFINALLY = "finally"
  214. )