const.go 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285
  1. /*
  2. * Public Domain Software
  3. *
  4. * I (Matthias Ladkau) am the author of the source code in this file.
  5. * I have placed the source code in this file in the public domain.
  6. *
  7. * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
  8. */
  9. /*
  10. Package parser contains a ECAL parser.
  11. Lexer for Source Text
  12. Lex() is a lexer function to convert a given search query into a list of tokens.
  13. Based on a talk by Rob Pike: Lexical Scanning in Go
  14. https://www.youtube.com/watch?v=HxaD_trXwRE
  15. The lexer's output is pushed into a channel which is consumed by the parser.
  16. This design enables the concurrent processing of the input text by lexer and
  17. parser.
  18. Parser
  19. Parse() is a parser which produces a parse tree from a given set of lexer tokens.
  20. Based on an article by Douglas Crockford: Top Down Operator Precedence
  21. http://crockford.com/javascript/tdop/tdop.html
  22. which is based on the ideas of Vaughan Pratt and his paper: Top Down Operator Precedence
  23. http://portal.acm.org/citation.cfm?id=512931
  24. https://tdop.github.io/
  25. ParseWithRuntime() parses a given input and decorates the resulting parse tree
  26. with runtime components which can be used to interpret the parsed query.
  27. */
  28. package parser
  29. /*
  30. LexTokenID represents a unique lexer token ID
  31. */
  32. type LexTokenID int
  33. /*
  34. Available meta data types
  35. */
  36. const (
  37. MetaDataPreComment = "MetaDataPreComment"
  38. MetaDataPostComment = "MetaDataPostComment"
  39. MetaDataGeneral = "MetaDataGeneral"
  40. )
  41. /*
  42. Available lexer token types
  43. */
  44. const (
  45. TokenError LexTokenID = iota // Lexing error token with a message as val
  46. TokenEOF // End-of-file token
  47. TokenANY // Unspecified token (used when building an AST from a Go map structure)
  48. TokenPRECOMMENT // Comment /* ... */
  49. TokenPOSTCOMMENT // Comment # ...
  50. // Value tokens
  51. TokenSTRING // String constant
  52. TokenNUMBER // Number constant
  53. TokenIDENTIFIER // Idendifier
  54. // Constructed tokens which are generated by the parser not the lexer
  55. TokenSTATEMENTS // A code block
  56. TokenFUNCCALL // A function call
  57. TokenCOMPACCESS // Access to a composition structure
  58. TokenLIST // List value
  59. TokenMAP // MAP value
  60. TokenPARAMS // Function parameters
  61. TokenGUARD // Conditional statements
  62. TOKENodeSYMBOLS // Used to separate symbols from other tokens in this list
  63. // Condition operators
  64. TokenGEQ
  65. TokenLEQ
  66. TokenNEQ
  67. TokenEQ
  68. TokenGT
  69. TokenLT
  70. // Grouping symbols
  71. TokenLPAREN
  72. TokenRPAREN
  73. TokenLBRACK
  74. TokenRBRACK
  75. TokenLBRACE
  76. TokenRBRACE
  77. // Separators
  78. TokenDOT
  79. TokenCOMMA
  80. TokenSEMICOLON
  81. // Grouping
  82. TokenCOLON
  83. TokenEQUAL
  84. // Arithmetic operators
  85. TokenPLUS
  86. TokenMINUS
  87. TokenTIMES
  88. TokenDIV
  89. TokenDIVINT
  90. TokenMODINT
  91. // Assignment statement
  92. TokenASSIGN
  93. TOKENodeKEYWORDS // Used to separate keywords from other tokens in this list
  94. // Import statement
  95. TokenIMPORT
  96. TokenAS
  97. // Sink definition
  98. TokenSINK
  99. TokenKINDMATCH
  100. TokenSCOPEMATCH
  101. TokenSTATEMATCH
  102. TokenPRIORITY
  103. TokenSUPPRESSES
  104. // Function definition
  105. TokenFUNC
  106. TokenRETURN
  107. // Boolean operators
  108. TokenAND
  109. TokenOR
  110. TokenNOT
  111. // Condition operators
  112. TokenLIKE
  113. TokenIN
  114. TokenHASPREFIX
  115. TokenHASSUFFIX
  116. TokenNOTIN
  117. // Constant terminals
  118. TokenFALSE
  119. TokenTRUE
  120. TokenNULL
  121. // Conditional statements
  122. TokenIF
  123. TokenELIF
  124. TokenELSE
  125. // Loop statements
  126. TokenFOR
  127. TokenBREAK
  128. TokenCONTINUE
  129. TokenENDLIST
  130. )
  131. /*
  132. IsValidTokenID check if a given token ID is valid.
  133. */
  134. func IsValidTokenID(value int) bool {
  135. return value < int(TokenENDLIST)
  136. }
  137. /*
  138. Available parser AST node types
  139. */
  140. const (
  141. NodeEOF = "EOF"
  142. NodeSTRING = "string" // String constant
  143. NodeNUMBER = "number" // Number constant
  144. NodeIDENTIFIER = "identifier" // Idendifier
  145. // Constructed tokens
  146. NodeSTATEMENTS = "statements" // List of statements
  147. NodeFUNCCALL = "funccall" // Function call
  148. NodeCOMPACCESS = "compaccess" // Composition structure access
  149. NodeLIST = "list" // List value
  150. NodeMAP = "map" // Map value
  151. NodePARAMS = "params" // Function parameters
  152. NodeGUARD = "guard" // Guard expressions for conditional statements
  153. // Condition operators
  154. NodeGEQ = ">="
  155. NodeLEQ = "<="
  156. NodeNEQ = "!="
  157. NodeEQ = "=="
  158. NodeGT = ">"
  159. NodeLT = "<"
  160. // Separators
  161. NodeKVP = "kvp" // Key-value pair
  162. NodePRESET = "preset" // Preset value
  163. // Arithmetic operators
  164. NodePLUS = "plus"
  165. NodeMINUS = "minus"
  166. NodeTIMES = "times"
  167. NodeDIV = "div"
  168. NodeMODINT = "modint"
  169. NodeDIVINT = "divint"
  170. // Assignment statement
  171. NodeASSIGN = ":="
  172. // Import statement
  173. NodeIMPORT = "import"
  174. // Sink definition
  175. NodeSINK = "sink"
  176. NodeKINDMATCH = "kindmatch"
  177. NodeSCOPEMATCH = "scopematch"
  178. NodeSTATEMATCH = "statematch"
  179. NodePRIORITY = "priority"
  180. NodeSUPPRESSES = "suppresses"
  181. // Function definition
  182. NodeFUNC = "function"
  183. NodeRETURN = "return"
  184. // Boolean operators
  185. NodeAND = "and"
  186. NodeOR = "or"
  187. NodeNOT = "not"
  188. // Condition operators
  189. NodeLIKE = "like"
  190. NodeIN = "in"
  191. NodeHASPREFIX = "hasprefix"
  192. NodeHASSUFFIX = "hassuffix"
  193. NodeNOTIN = "notin"
  194. // Constant terminals
  195. NodeTRUE = "true"
  196. NodeFALSE = "false"
  197. NodeNULL = "null"
  198. // Conditional statements
  199. NodeIF = "if"
  200. // Loop statements
  201. NodeLOOP = "loop"
  202. NodeBREAK = "break"
  203. NodeCONTINUE = "continue"
  204. )