const.go 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263
  1. /*
  2. * Public Domain Software
  3. *
  4. * I (Matthias Ladkau) am the author of the source code in this file.
  5. * I have placed the source code in this file in the public domain.
  6. *
  7. * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
  8. */
  9. /*
  10. Package parser contains a ECAL parser.
  11. Lexer for Source Text
  12. Lex() is a lexer function to convert a given search query into a list of tokens.
  13. Based on a talk by Rob Pike: Lexical Scanning in Go
  14. https://www.youtube.com/watch?v=HxaD_trXwRE
  15. The lexer's output is pushed into a channel which is consumed by the parser.
  16. This design enables the concurrent processing of the input text by lexer and
  17. parser.
  18. Parser
  19. Parse() is a parser which produces a parse tree from a given set of lexer tokens.
  20. Based on an article by Douglas Crockford: Top Down Operator Precedence
  21. http://crockford.com/javascript/tdop/tdop.html
  22. which is based on the ideas of Vaughan Pratt and his paper: Top Down Operator Precedence
  23. http://portal.acm.org/citation.cfm?id=512931
  24. https://tdop.github.io/
  25. ParseWithRuntime() parses a given input and decorates the resulting parse tree
  26. with runtime components which can be used to interpret the parsed query.
  27. */
  28. package parser
  29. /*
  30. LexTokenID represents a unique lexer token ID
  31. */
  32. type LexTokenID int
  33. /*
  34. Available lexer token types
  35. */
  36. const (
  37. TokenError LexTokenID = iota // Lexing error token with a message as val
  38. TokenEOF // End-of-file token
  39. TokenANY // Unspecified token (used when building an AST from a Go map structure)
  40. TokenCOMMENT // Comment
  41. TokenSTRING // String constant
  42. TokenNUMBER // Number constant
  43. TokenIDENTIFIER // Idendifier
  44. // Constructed tokens which are generated by the parser not the lexer
  45. TokenSTATEMENTS // A code block
  46. TokenLIST // List value
  47. TokenMAP // MAP value
  48. TokenGUARD // Guard expressions for conditional statements
  49. TOKENodeSYMBOLS // Used to separate symbols from other tokens in this list
  50. // Condition operators
  51. TokenGEQ
  52. TokenLEQ
  53. TokenNEQ
  54. TokenEQ
  55. TokenGT
  56. TokenLT
  57. // Grouping symbols
  58. TokenLPAREN
  59. TokenRPAREN
  60. TokenLBRACK
  61. TokenRBRACK
  62. TokenLBRACE
  63. TokenRBRACE
  64. // Separators
  65. TokenDOT
  66. TokenCOMMA
  67. TokenCOLON
  68. TokenSEMICOLON
  69. // Arithmetic operators
  70. TokenPLUS
  71. TokenMINUS
  72. TokenTIMES
  73. TokenDIV
  74. TokenDIVINT
  75. TokenMODINT
  76. // Assignment statement
  77. TokenASSIGN
  78. // Data structure access
  79. TokenACCESS
  80. // The colon '' has a context specific meaning and is checked by the parser
  81. TOKENodeKEYWORDS // Used to separate keywords from other tokens in this list
  82. // Sink definition
  83. TokenSINK
  84. TokenKINDMATCH
  85. TokenSCOPEMATCH
  86. TokenSTATEMATCH
  87. TokenPRIORITY
  88. TokenSUPPRESSES
  89. // Function definition
  90. TokenFUNC
  91. // Boolean operators
  92. TokenAND
  93. TokenOR
  94. TokenNOT
  95. // Condition operators
  96. TokenLIKE
  97. TokenIN
  98. TokenHASPREFIX
  99. TokenHASSUFFIX
  100. TokenNOTIN
  101. // Constant terminals
  102. TokenFALSE
  103. TokenTRUE
  104. TokenNULL
  105. // Conditional statements
  106. TokenIF
  107. TokenELIF
  108. TokenELSE
  109. // Loop statements
  110. TokenFOR
  111. TokenBREAK
  112. TokenCONTINUE
  113. TokenENDLIST
  114. )
  115. /*
  116. IsValidTokenID check if a given token ID is valid.
  117. */
  118. func IsValidTokenID(value int) bool {
  119. return value < int(TokenENDLIST)
  120. }
  121. /*
  122. Available parser AST node types
  123. */
  124. const (
  125. NodeEOF = "EOF"
  126. NodeCOMMENT = "comment" // Comment
  127. NodeSTRING = "string" // String constant
  128. NodeNUMBER = "number" // Number constant
  129. NodeIDENTIFIER = "identifier" // Idendifier
  130. // Constructed tokens
  131. NodeSTATEMENTS = "statements" // List of statements
  132. // Assignment statement
  133. NodeASSIGN = ":="
  134. // Arithmetic operators
  135. NodePLUS = "plus"
  136. NodeMINUS = "minus"
  137. NodeTIMES = "times"
  138. NodeDIV = "div"
  139. NodeMODINT = "modint"
  140. NodeDIVINT = "divint"
  141. // Boolean operators
  142. NodeOR = "or"
  143. NodeAND = "and"
  144. NodeNOT = "not"
  145. // Condition operators
  146. NodeLIKE = "like"
  147. NodeIN = "in"
  148. NodeHASPREFIX = "hasprefix"
  149. NodeHASSUFFIX = "hassuffix"
  150. NodeNOTIN = "notin"
  151. NodeGEQ = ">="
  152. NodeLEQ = "<="
  153. NodeNEQ = "!="
  154. NodeEQ = "=="
  155. NodeGT = ">"
  156. NodeLT = "<"
  157. // Constants
  158. NodeTRUE = "true"
  159. NodeFALSE = "false"
  160. NodeNULL = "null"
  161. /*
  162. NodeLIST = "list" // List value
  163. NodeMAP = "map" // Map value
  164. NodeGUARD = "guard" // Guard expressions for conditional statements
  165. // Map entries
  166. NodeMAPENTRY = "entry" // Map entry value
  167. // Function call statement
  168. NodeFUNCCALL = "funccall"
  169. // Data structure access
  170. NodeACCESS = "access"
  171. // Sink definition
  172. NodeSINK = "sink"
  173. NodeKINDMATCH = "kindmatch"
  174. NodeSCOPEMATCH = "scopematch"
  175. NodeSTATEMATCH = "statematch"
  176. NodePRIORITY = "priority"
  177. NodeSUPPRESSES = "suppresses"
  178. // Block statements
  179. NodeCOND = "cond"
  180. NodeLOOP = "loop"
  181. // Single statements
  182. NodeBREAK = "break"
  183. NodeCONTINUE = "continue"
  184. */
  185. )