query.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. package v1
  11. import (
  12. "encoding/json"
  13. "fmt"
  14. "net/http"
  15. "strings"
  16. "time"
  17. "devt.de/krotik/common/datautil"
  18. "devt.de/krotik/common/stringutil"
  19. "devt.de/krotik/eliasdb/api"
  20. "devt.de/krotik/eliasdb/eql"
  21. "devt.de/krotik/eliasdb/graph/data"
  22. )
  23. /*
  24. ResultCacheMaxSize is the maximum size for the result cache
  25. */
  26. var ResultCacheMaxSize uint64
  27. /*
  28. ResultCacheMaxAge is the maximum age a result cache entry can have in seconds
  29. */
  30. var ResultCacheMaxAge int64
  31. /*
  32. ResultCache is a cache for result sets (by default no expiry and no limit)
  33. */
  34. var ResultCache *datautil.MapCache
  35. /*
  36. idCount is an ID counter for results
  37. */
  38. var idCount = time.Now().Unix()
  39. /*
  40. EndpointQuery is the query endpoint URL (rooted). Handles everything under query/...
  41. */
  42. const EndpointQuery = api.APIRoot + APIv1 + "/query/"
  43. /*
  44. QueryEndpointInst creates a new endpoint handler.
  45. */
  46. func QueryEndpointInst() api.RestEndpointHandler {
  47. // Init the result cache if necessary
  48. if ResultCache == nil {
  49. ResultCache = datautil.NewMapCache(ResultCacheMaxSize, ResultCacheMaxAge)
  50. }
  51. return &queryEndpoint{}
  52. }
  53. /*
  54. Handler object for search queries.
  55. */
  56. type queryEndpoint struct {
  57. *api.DefaultEndpointHandler
  58. }
  59. /*
  60. HandleGET handles a search query REST call.
  61. */
  62. func (eq *queryEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
  63. var err error
  64. // Check parameters
  65. if !checkResources(w, resources, 1, 1, "Need a partition") {
  66. return
  67. }
  68. // Get partition
  69. part := resources[0]
  70. // Get limit parameter; -1 if not set
  71. limit, ok := queryParamPosNum(w, r, "limit")
  72. if !ok {
  73. return
  74. }
  75. // Get offset parameter; -1 if not set
  76. offset, ok := queryParamPosNum(w, r, "offset")
  77. if !ok {
  78. return
  79. }
  80. // Get groups parameter
  81. gs := r.URL.Query().Get("groups")
  82. showGroups := gs != ""
  83. // See if a result ID was given
  84. resID := r.URL.Query().Get("rid")
  85. if resID != "" {
  86. res, ok := ResultCache.Get(resID)
  87. if !ok {
  88. http.Error(w, "Unknown result ID (rid parameter)", http.StatusBadRequest)
  89. return
  90. }
  91. err = eq.writeResultData(w, res.(*APISearchResult), part, resID, offset, limit, showGroups)
  92. } else {
  93. var res eql.SearchResult
  94. // Run the query
  95. query := r.URL.Query().Get("q")
  96. if query == "" {
  97. http.Error(w, "Missing query (q parameter)", http.StatusBadRequest)
  98. return
  99. }
  100. res, err = eql.RunQuery(stringutil.CreateDisplayString(part)+" query",
  101. part, query, api.GM)
  102. if err == nil {
  103. sres := &APISearchResult{res, nil}
  104. // Make sure the result has a primary node column
  105. _, err = sres.GetPrimaryNodeColumn()
  106. if err != nil {
  107. http.Error(w, err.Error(), http.StatusBadRequest)
  108. return
  109. }
  110. // Store the result in the cache
  111. resID = genID()
  112. ResultCache.Put(resID, sres)
  113. err = eq.writeResultData(w, sres, part, resID, offset, limit, showGroups)
  114. }
  115. }
  116. if err != nil {
  117. http.Error(w, err.Error(), http.StatusInternalServerError)
  118. }
  119. }
  120. /*
  121. writeResultData writes result data for the client.
  122. */
  123. func (eq *queryEndpoint) writeResultData(w http.ResponseWriter, res *APISearchResult,
  124. part string, resID string, offset int, limit int, showGroups bool) error {
  125. var err error
  126. // Write out the data
  127. header := res.Header()
  128. ret := json.NewEncoder(w)
  129. resdata := make(map[string]interface{})
  130. // Count total selections
  131. sels := res.Selections()
  132. totalSels := 0
  133. for _, s := range sels {
  134. if s {
  135. totalSels++
  136. }
  137. }
  138. resdata["total_selections"] = totalSels
  139. rows := res.Rows()
  140. srcs := res.RowSources()
  141. if limit == -1 && offset == -1 {
  142. resdata["rows"] = rows
  143. resdata["sources"] = srcs
  144. resdata["selections"] = sels
  145. } else {
  146. if offset > 0 {
  147. if offset >= len(rows) {
  148. return fmt.Errorf("Offset exceeds available rows")
  149. }
  150. rows = rows[offset:]
  151. srcs = srcs[offset:]
  152. sels = sels[offset:]
  153. }
  154. if limit != -1 && limit < len(rows) {
  155. rows = rows[:limit]
  156. srcs = srcs[:limit]
  157. sels = sels[:limit]
  158. }
  159. resdata["rows"] = rows
  160. resdata["sources"] = srcs
  161. resdata["selections"] = sels
  162. }
  163. // Write out result header
  164. resdataHeader := make(map[string]interface{})
  165. resdata["header"] = resdataHeader
  166. resdataHeader["labels"] = header.Labels()
  167. resdataHeader["format"] = header.Format()
  168. resdataHeader["data"] = header.Data()
  169. pk := header.PrimaryKind()
  170. resdataHeader["primary_kind"] = pk
  171. if showGroups {
  172. groupList := make([][]string, 0, len(srcs))
  173. if len(srcs) > 0 {
  174. var col int
  175. // Get column for primary kind
  176. col, err = res.GetPrimaryNodeColumn()
  177. // Lookup groups for nodes
  178. for _, s := range resdata["sources"].([][]string) {
  179. if err == nil {
  180. var nodes []data.Node
  181. groups := make([]string, 0, 3)
  182. key := strings.Split(s[col], ":")[2]
  183. nodes, _, err = api.GM.TraverseMulti(part, key, pk,
  184. ":::"+eql.GroupNodeKind, false)
  185. if err == nil {
  186. for _, n := range nodes {
  187. groups = append(groups, n.Key())
  188. }
  189. }
  190. groupList = append(groupList, groups)
  191. }
  192. }
  193. }
  194. resdata["groups"] = groupList
  195. }
  196. if err == nil {
  197. // Set response header values
  198. w.Header().Add(HTTPHeaderTotalCount, fmt.Sprint(res.RowCount()))
  199. w.Header().Add(HTTPHeaderCacheID, resID)
  200. w.Header().Set("content-type", "application/json; charset=utf-8")
  201. ret.Encode(resdata)
  202. }
  203. return err
  204. }
  205. /*
  206. SwaggerDefs is used to describe the endpoint in swagger.
  207. */
  208. func (eq *queryEndpoint) SwaggerDefs(s map[string]interface{}) {
  209. // Add query paths
  210. s["paths"].(map[string]interface{})["/v1/query/{partition}"] = map[string]interface{}{
  211. "get": map[string]interface{}{
  212. "summary": "Run EQL queries to query the EliasDB datastore.",
  213. "description": "The query endpoint should be used to run EQL search " +
  214. "queries against partitions. The return value is always a list " +
  215. "(even if there is only a single entry). A query result gets an " +
  216. "ID and is stored in a cache. The ID is returned in the X-Cache-Id " +
  217. "header. Subsequent requests for the same result can use the ID instead of a query.",
  218. "produces": []string{
  219. "text/plain",
  220. "application/json",
  221. },
  222. "parameters": []map[string]interface{}{
  223. {
  224. "name": "partition",
  225. "in": "path",
  226. "description": "Partition to query.",
  227. "required": true,
  228. "type": "string",
  229. },
  230. {
  231. "name": "q",
  232. "in": "query",
  233. "description": "URL encoded query to execute.",
  234. "required": false,
  235. "type": "string",
  236. },
  237. {
  238. "name": "rid",
  239. "in": "query",
  240. "description": "Result ID to retrieve from the result cache.",
  241. "required": false,
  242. "type": "number",
  243. "format": "integer",
  244. },
  245. {
  246. "name": "limit",
  247. "in": "query",
  248. "description": "How many list items to return.",
  249. "required": false,
  250. "type": "number",
  251. "format": "integer",
  252. },
  253. {
  254. "name": "offset",
  255. "in": "query",
  256. "description": "Offset in the dataset.",
  257. "required": false,
  258. "type": "number",
  259. "format": "integer",
  260. },
  261. {
  262. "name": "groups",
  263. "in": "query",
  264. "description": "Include group information in the result if set to any value.",
  265. "required": false,
  266. "type": "number",
  267. "format": "integer",
  268. },
  269. },
  270. "responses": map[string]interface{}{
  271. "200": map[string]interface{}{
  272. "description": "A query result",
  273. "schema": map[string]interface{}{
  274. "$ref": "#/definitions/QueryResult",
  275. },
  276. },
  277. "default": map[string]interface{}{
  278. "description": "Error response",
  279. "schema": map[string]interface{}{
  280. "$ref": "#/definitions/Error",
  281. },
  282. },
  283. },
  284. },
  285. }
  286. // Add QueryResult to definitions
  287. s["definitions"].(map[string]interface{})["QueryResult"] = map[string]interface{}{
  288. "type": "object",
  289. "properties": map[string]interface{}{
  290. "header": map[string]interface{}{
  291. "description": "Header for the query result.",
  292. "type": "object",
  293. "properties": map[string]interface{}{
  294. "labels": map[string]interface{}{
  295. "description": "All column labels of the search result.",
  296. "type": "array",
  297. "items": map[string]interface{}{
  298. "description": "Column label.",
  299. "type": "string",
  300. },
  301. },
  302. "format": map[string]interface{}{
  303. "description": "All column format definitions of the search result.",
  304. "type": "array",
  305. "items": map[string]interface{}{
  306. "description": "Column format as specified in the show format (e.g. text).",
  307. "type": "string",
  308. },
  309. },
  310. "data": map[string]interface{}{
  311. "description": "The data which is displayed in each column of the search result.",
  312. "type": "array",
  313. "items": map[string]interface{}{
  314. "description": "Data source for the column (e.g. 1:n:name - Name of starting nodes, 3:e:key - Key of edge traversed in the second traversal).",
  315. "type": "string",
  316. },
  317. },
  318. },
  319. },
  320. "rows": map[string]interface{}{
  321. "description": "Rows of the query result.",
  322. "type": "array",
  323. "items": map[string]interface{}{
  324. "description": "Columns of a row of the query result.",
  325. "type": "array",
  326. "items": map[string]interface{}{
  327. "description": "A single cell of the query result (string, integer or null).",
  328. "type": "object",
  329. },
  330. },
  331. },
  332. "sources": map[string]interface{}{
  333. "description": "Data sources of the query result.",
  334. "type": "array",
  335. "items": map[string]interface{}{
  336. "description": "Columns of a row of the query result.",
  337. "type": "array",
  338. "items": map[string]interface{}{
  339. "description": "Data source of a single cell of the query result.",
  340. "type": "string",
  341. },
  342. },
  343. },
  344. "groups": map[string]interface{}{
  345. "description": "Group names for each row.",
  346. "type": "array",
  347. "items": map[string]interface{}{
  348. "description": " Groups of the primary kind node.",
  349. "type": "array",
  350. "items": map[string]interface{}{
  351. "description": "Group name.",
  352. "type": "string",
  353. },
  354. },
  355. },
  356. "selections": map[string]interface{}{
  357. "description": "List of row selections.",
  358. "type": "array",
  359. "items": map[string]interface{}{
  360. "description": "Row selection.",
  361. "type": "boolean",
  362. },
  363. },
  364. "total_selections": map[string]interface{}{
  365. "description": "Number of total selections.",
  366. "type": "number",
  367. "format": "integer",
  368. },
  369. },
  370. }
  371. // Add generic error object to definition
  372. s["definitions"].(map[string]interface{})["Error"] = map[string]interface{}{
  373. "description": "A human readable error mesage.",
  374. "type": "string",
  375. }
  376. }
  377. /*
  378. genID generates a unique ID.
  379. */
  380. func genID() string {
  381. idCount++
  382. return fmt.Sprint(idCount)
  383. }
  384. /*
  385. APISearchResult is a search result maintained by the API. It embeds
  386. */
  387. type APISearchResult struct {
  388. eql.SearchResult // Normal eql search result
  389. selections []bool // Selections of the result
  390. }
  391. /*
  392. GetPrimaryNodeColumn determines the first primary node column.
  393. */
  394. func (r *APISearchResult) GetPrimaryNodeColumn() (int, error) {
  395. var err error
  396. pk := r.Header().PrimaryKind()
  397. col := -1
  398. rs := r.RowSources()
  399. if len(rs) > 0 {
  400. for i, scol := range rs[0] {
  401. scolParts := strings.Split(scol, ":")
  402. if len(scolParts) > 1 && pk == scolParts[1] {
  403. col = i
  404. }
  405. }
  406. }
  407. if col == -1 {
  408. err = fmt.Errorf("Could not determine key of primary node - query needs a primary expression")
  409. }
  410. return col, err
  411. }
  412. /*
  413. Selections returns all current selections.
  414. */
  415. func (r *APISearchResult) Selections() []bool {
  416. r.refreshSelection()
  417. return r.selections
  418. }
  419. /*
  420. SetSelection sets a new selection.
  421. */
  422. func (r *APISearchResult) SetSelection(line int, selection bool) {
  423. r.refreshSelection()
  424. if line < len(r.selections) {
  425. r.selections[line] = selection
  426. }
  427. }
  428. /*
  429. AllSelection selects all rows.
  430. */
  431. func (r *APISearchResult) AllSelection() {
  432. r.refreshSelection()
  433. for i := 0; i < len(r.selections); i++ {
  434. r.selections[i] = true
  435. }
  436. }
  437. /*
  438. NoneSelection selects none rows.
  439. */
  440. func (r *APISearchResult) NoneSelection() {
  441. r.refreshSelection()
  442. for i := 0; i < len(r.selections); i++ {
  443. r.selections[i] = false
  444. }
  445. }
  446. /*
  447. InvertSelection inverts the current selection.
  448. */
  449. func (r *APISearchResult) InvertSelection() {
  450. r.refreshSelection()
  451. for i := 0; i < len(r.selections); i++ {
  452. r.selections[i] = !r.selections[i]
  453. }
  454. }
  455. /*
  456. refreshSelection reallocates the selection array if necessary.
  457. */
  458. func (r *APISearchResult) refreshSelection() {
  459. l := r.SearchResult.RowCount()
  460. if len(r.selections) != l {
  461. origSelections := r.selections
  462. // There is a difference between the selections array and the row
  463. // count we need to resize
  464. r.selections = make([]bool, l)
  465. for i, s := range origSelections {
  466. if i < l {
  467. r.selections[i] = s
  468. }
  469. }
  470. }
  471. }