blob.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. /*
  11. Package v1 contains EliasDB REST API Version 1.
  12. Binary Blob control endpoint
  13. /blob
  14. The blob endpoint can be used to store and retrieve binary data to/from automatically
  15. allocated storage locatons.
  16. A new binary blob can be stored by sending a POST request. The body should
  17. be the binary data to store. The response should have the following structure:
  18. {
  19. id : <ID of the stored binary blob>
  20. }
  21. /blob/<id>
  22. GET requests can be used to retrieve a binary blobs with a specific ID. Binary blobs
  23. can be updated by sending a PUT request and removed by sending a DELETE request.
  24. Cluster control endpoint
  25. /cluster
  26. The cluster endpoint returns cluster state specific information. A GET request
  27. returns the cluster state info as a key-value map:
  28. {
  29. <stateinfo key> : <info value>,
  30. ...
  31. }
  32. /cluster/join
  33. An instance can join an existing cluster by sending a PUT request to the join
  34. endpoint. The body should have the following datastructure:
  35. {
  36. name : <Name of an existing cluster member>,
  37. netaddr : <Network address of an existing cluster member>
  38. }
  39. /cluster/eject
  40. A cluster member can eject another cluster member or itself by sending a PUT
  41. request to the eject endpoint. The body should have the following datastructure:
  42. {
  43. name : <Name the cluster member to eject>,
  44. }
  45. /cluster/ping
  46. An instance can ping another instance (provided the secret is correct). Cluster
  47. membership is not required for this command. The body should have the following datastructure:
  48. {
  49. name : <Name of an existing instance>,
  50. netaddr : <Network address of an existing instance>
  51. }
  52. /cluster/memberinfos
  53. The memberinfos endpoint returns the static member info of every known cluster
  54. member. If a member is not reachable its info contains a single key-value pair with
  55. the key error and an error message as value. A GET request returns the member
  56. info of every member as a key-value map:
  57. {
  58. <memberinfo key> : <memberinfo value>,
  59. ...
  60. }
  61. /cluster/log
  62. Returns the latest cluster related log messages. A DELETE call will clear
  63. the current log.
  64. EQL parser endpoint
  65. /eql
  66. The EQL endpoint provides direct access to the EQL parser. It can be used
  67. to parse a given EQL query into an Abstract Syntax Tree or pretty print a
  68. given Abstract Syntax Tree into an EQL query.
  69. A query can be parsed into an Abstract Syntax Tree by sending a POST request. The
  70. body should have the following format:
  71. {
  72. query : <Query to parse>
  73. }
  74. Returns a JSON structure or an error message.
  75. {
  76. ast : <AST of the given query>
  77. }
  78. An Abstract Syntax Tree can be pretty printed into a query by sending a POST request.
  79. The body should have the following format:
  80. {
  81. ast : <AST to pretty print>
  82. }
  83. Returns a JSON structure or an error message.
  84. {
  85. query : <Pretty printed query>
  86. }
  87. Graph request endpoint
  88. /graph
  89. The graph endpoint is the main entry point to send and request graph data.
  90. Data can be send by using POST and PUT requests. POST will store
  91. data in the datastore and always overwrite any existing data. PUT requests on
  92. nodes will only update the given attributes. PUT requests on edges are handled
  93. equally to POST requests. Data can be deleted using DELETE requests. The data
  94. structure for DELETE requests requires only the key and kind attributes.
  95. A PUT, POST or DELETE request should be send to one of the following
  96. endpoints:
  97. /graph/<partition>
  98. A graph with the following datastructure:
  99. {
  100. nodes : [ { <attr> : <value> }, ... ],
  101. edges : [ { <attr> : <value> }, ... ]
  102. }
  103. /graph/<partition>/n
  104. A list of nodes:
  105. [ { <attr> : <value> }, ... ]
  106. /graph/<partition>/e
  107. A list of edges:
  108. [ { <attr> : <value> }, ... ]
  109. GET requests can be used to query single or a series of nodes. The endpoints
  110. support the limit and offset parameters for lists:
  111. limit - How many list items to return
  112. offset - Offset in the dataset (0 to <total count>-1)
  113. The total number of entries is returned in the X-Total-Count header when
  114. a list is returned.
  115. /graph/<partition>/n/<node kind>/[node key]/[traversal spec]
  116. /graph/<partition>/e/<edge kind>/<edge key>
  117. The return data is a list of objects unless a specific node / edge or a traversal
  118. from a specific node is requested. Each object in the list models a node or edge.
  119. [{
  120. key : <value>,
  121. ...
  122. }]
  123. If a specifc object is requested then the return data is a single object.
  124. {
  125. key : <value>,
  126. ...
  127. }
  128. Traversals return two lists containing traversed nodes and edges. The traversal
  129. endpoint does NOT support limit and offset parameters. Also the X-Total-Count
  130. header is not set.
  131. [
  132. [ <traversed nodes> ], [ <traversed edges> ]
  133. ]
  134. Index query endpoint
  135. /index
  136. The index query endpoint should be used to run index search queries against
  137. partitions. Index queries look for words or phrases on all nodes of a given
  138. node kind.
  139. A phrase query finds all nodes/edges where an attribute contains a
  140. certain phrase. A request url which runs a new phrase search should be of the
  141. following form:
  142. /index/<partition>/n/<node kind>?phrase=<phrase>&attr=<attribute>
  143. /index/<partition>/e/<edge kind>?phrase=<phrase>&attr=<attribute>
  144. The return data is a list of node keys:
  145. [ <node key1>, <node key2>, ... ]
  146. A word query finds all nodes/edges where an attribute contains a certain word.
  147. A request url which runs a new word search should be of the following form:
  148. /index/<partition>/n/<node kind>?word=<word>&attr=<attribute>
  149. /index/<partition>/e/<edge kind>?word=<word>&attr=<attribute>
  150. The return data is a map which maps node key to a list of word positions:
  151. {
  152. key : [ <pos1>, <pos2>, ... ],
  153. ...
  154. }
  155. A value search finds all nodes/edges where an attribute has a certain value.
  156. A request url which runs a new value search should be of the following form:
  157. /index/<partition>/n/<node kind>?value=<value>&attr=<attribute>
  158. /index/<partition>/e/<edge kind>?value=<value>&attr=<attribute>
  159. The return data is a list of node keys:
  160. [ <node key1>, <node key2>, ... ]
  161. Find query endpoint
  162. /find
  163. The find query endpoint is a simplified index query which looks up nodes
  164. in all partitions which do not start with a _ character. It either searches
  165. for a word / phrase or an exact value on all available attributes.
  166. A phrase query finds all nodes/edges where an attribute contains a
  167. certain phrase. A request url should be of the following form:
  168. /find?text=<word or phrase value>
  169. /find?value=<exact value>
  170. The return data is a map of partitions to node kinds to a list of nodes:
  171. {
  172. <partition> : {
  173. <kind> : [ { node1 }, { node2 }, ... ]
  174. ...
  175. }
  176. ...
  177. }
  178. GraphQL request endpoint
  179. /graphql
  180. /graphql-query
  181. The GraphQL endpoints execute GraphQL queries on EliasDB's datastore. The
  182. query endpoint supports only read-queries (i.e. no mutations). EliasDB supports
  183. only executable definitions and introspection (i.e. no type system validation).
  184. General database information endpoint
  185. /info
  186. The info endpoint returns general database information such as known
  187. node kinds, known attributes, etc ..
  188. The return data is a key-value map:
  189. {
  190. <info name> : <info value>,
  191. ...
  192. }
  193. /info/kind/<kind>
  194. The node kind info endpoint returns general information about a known node or
  195. edge kind such as known attributes or known edges.
  196. Query endpoint
  197. /query
  198. The query endpoint should be used to run EQL search queries against partitions.
  199. The return value is always a list (even if there is only a single entry).
  200. A query result gets an ID and is stored in a cache. The ID is returned in the
  201. X-Cache-Id header. Subsequent requests for the same result can use the ID
  202. instead of a query.
  203. The endpoint supports the optional limit, offset and groups parameter:
  204. limit - How many list items to return
  205. offset - Offset in the dataset
  206. groups - If set then group information are included in the result
  207. (depending on the result size this can be an expensive call)
  208. The total number of entries in the result is returned in the X-Total-Count header.
  209. A request url which runs a new query should be of the following form:
  210. /query/<partition>?q=<query>
  211. /query/<partition>?rid=<result id>
  212. The return data is a result object:
  213. {
  214. header : {
  215. labels : All column labels of the search result.
  216. format : All column format definitions of the search result.
  217. data : The data which is displayed in each column of the search result.
  218. (e.g. 1:n:name - Name of starting nodes,
  219. 3:e:key - Key of edge traversed in the second traversal)
  220. primary_kind : The primary kind of the search result.
  221. },
  222. rows : [ [ <col0>, <col1>, ... ] ],
  223. sources : [ [ <src col0>, <src col1>, ... ] ],
  224. selections : [ <row selected> ],
  225. total_selections : <number of total selections>
  226. groups : [ [ <groups of row0> ], [ <groups of row1> ] ... ]
  227. }
  228. Query result endpoint
  229. /queryresult
  230. The query result endpoint is used to run operations on query results.
  231. The quickfilter endpoint (GET) is used to determine the most frequent used values
  232. in a particular result column.
  233. /queryresult/<rid>/quickfilter/<column>?limit=<max result items>
  234. The optional limit parameter can be used to limit the result items. The return
  235. data is a simple object:
  236. {
  237. values : [ <value1>, ... ],
  238. frequencies : [ <frequency1>, ... ]
  239. }
  240. /queryresult/<rid>/select
  241. The select endpoint (GET) returns the (primary) nodes which are currently
  242. selected. The primary node of each row is usually the node from which
  243. the query started, when constructing the row of the result (unless the
  244. primary keyword was used). The return data is a simple object:
  245. {
  246. keys : [ <key of selected node1>, ... ],
  247. kinds : [ <kind of selected node1>, ... ]
  248. }
  249. /queryresult/<rid>/select/<row>
  250. The select endpoint with the row parameter (PUT) is used to select
  251. single or multiple rows of a query result. The row parameter can either
  252. be a positive number or 'all', 'none' or 'invert'. Returns the new
  253. number of total selections.
  254. /queryresult/<rid>/groupselected
  255. The groupselected endpoint returns the groups which contain the selected
  256. (primary) nodes based on the currently selected rows. The primary node
  257. of each row is usually the node from which the query started, when
  258. constructing the row of the result (unless the primary keyword was used).
  259. The return data is a simple object:
  260. {
  261. groups : [ <group1>, ... ],
  262. keys : [ [ <keys of selected nodes in group1> ], ... ],
  263. kinds : [ [ <kinds of selected nodes in group1> ], ... ]
  264. }
  265. The state can be set by sending it to the endpoint via a POST request.
  266. /queryresult/<rid>/groupselected/<name>
  267. The groupselected endpoint with a group name adds (PUT) or removes (DELETE) all
  268. selected nodes to/from the given (existing) group.
  269. /queryresult/<rid>/csv
  270. The csv endpoint returns the search result as CSV string.
  271. */
  272. package v1
  273. import (
  274. "bytes"
  275. "encoding/json"
  276. "fmt"
  277. "net/http"
  278. "strconv"
  279. "devt.de/krotik/eliasdb/api"
  280. "devt.de/krotik/eliasdb/storage"
  281. )
  282. /*
  283. StorageSuffixBlob is the suffix for binary blob storage
  284. */
  285. const StorageSuffixBlob = ".blob"
  286. /*
  287. EndpointBlob is the blob endpoint URL (rooted). Handles everything under blob/...
  288. */
  289. const EndpointBlob = api.APIRoot + APIv1 + "/blob/"
  290. /*
  291. BlobEndpointInst creates a new endpoint handler.
  292. */
  293. func BlobEndpointInst() api.RestEndpointHandler {
  294. return &blobEndpoint{}
  295. }
  296. /*
  297. Handler object for blob operations.
  298. */
  299. type blobEndpoint struct {
  300. *api.DefaultEndpointHandler
  301. }
  302. /*
  303. HandleGET handles REST calls to retrieve binary data.
  304. */
  305. func (be *blobEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
  306. var res interface{}
  307. var ret []byte
  308. // Check parameters
  309. if !checkResources(w, resources, 2, 2, "Need a partition and a specific data ID") {
  310. return
  311. }
  312. loc, err := strconv.ParseUint(resources[1], 10, 64)
  313. if err != nil {
  314. http.Error(w, fmt.Sprint("Could not decode data ID: ", err.Error()),
  315. http.StatusBadRequest)
  316. return
  317. }
  318. sm := api.GS.StorageManager(resources[0]+StorageSuffixBlob, false)
  319. if sm != nil {
  320. res, err = sm.FetchCached(loc)
  321. if err == storage.ErrNotInCache {
  322. err = sm.Fetch(loc, &ret)
  323. } else if err == nil && res != nil {
  324. ret = res.([]byte)
  325. }
  326. }
  327. // Write data
  328. w.Header().Set("content-type", "application/octet-stream")
  329. w.Write(ret)
  330. }
  331. /*
  332. HandlePOST handles a REST call to store new binary data.
  333. */
  334. func (be *blobEndpoint) HandlePOST(w http.ResponseWriter, r *http.Request, resources []string) {
  335. var buf bytes.Buffer
  336. // Check parameters
  337. if !checkResources(w, resources, 1, 1, "Need a partition") {
  338. return
  339. }
  340. sm := api.GS.StorageManager(resources[0]+StorageSuffixBlob, true)
  341. // Use a memory buffer to read send data
  342. buf.ReadFrom(r.Body)
  343. loc, err := sm.Insert(buf.Bytes())
  344. if err != nil {
  345. http.Error(w, err.Error(), http.StatusInternalServerError)
  346. return
  347. }
  348. sm.Flush()
  349. // Write data
  350. w.Header().Set("content-type", "application/json; charset=utf-8")
  351. ret := json.NewEncoder(w)
  352. ret.Encode(map[string]interface{}{
  353. "id": loc,
  354. })
  355. }
  356. /*
  357. HandlePUT handles a REST call to update existing binary data.
  358. */
  359. func (be *blobEndpoint) HandlePUT(w http.ResponseWriter, r *http.Request, resources []string) {
  360. var buf bytes.Buffer
  361. // Check parameters
  362. if !checkResources(w, resources, 2, 2, "Need a partition and a specific data ID") {
  363. return
  364. }
  365. loc, err := strconv.ParseUint(resources[1], 10, 64)
  366. if err != nil {
  367. http.Error(w, fmt.Sprint("Could not decode data ID: ", err.Error()), http.StatusBadRequest)
  368. return
  369. }
  370. sm := api.GS.StorageManager(resources[0]+StorageSuffixBlob, false)
  371. if sm != nil {
  372. // Use a memory buffer to read send data
  373. buf.ReadFrom(r.Body)
  374. if err := sm.Update(loc, buf.Bytes()); err != nil {
  375. http.Error(w, err.Error(), http.StatusInternalServerError)
  376. return
  377. }
  378. sm.Flush()
  379. }
  380. }
  381. /*
  382. HandleDELETE handles a REST call to remove existing binary data.
  383. */
  384. func (be *blobEndpoint) HandleDELETE(w http.ResponseWriter, r *http.Request, resources []string) {
  385. // Check parameters
  386. if !checkResources(w, resources, 2, 2, "Need a partition and a specific data ID") {
  387. return
  388. }
  389. loc, err := strconv.ParseUint(resources[1], 10, 64)
  390. if err != nil {
  391. http.Error(w, fmt.Sprint("Could not decode data ID: ", err.Error()), http.StatusBadRequest)
  392. return
  393. }
  394. sm := api.GS.StorageManager(resources[0]+StorageSuffixBlob, false)
  395. if sm != nil {
  396. if err := sm.Free(loc); err != nil {
  397. http.Error(w, err.Error(), http.StatusInternalServerError)
  398. return
  399. }
  400. sm.Flush()
  401. }
  402. }
  403. /*
  404. SwaggerDefs is used to describe the endpoint in swagger.
  405. */
  406. func (be *blobEndpoint) SwaggerDefs(s map[string]interface{}) {
  407. idParams := []map[string]interface{}{
  408. {
  409. "name": "id",
  410. "in": "path",
  411. "description": "ID of the binary blob.",
  412. "required": true,
  413. "type": "string",
  414. },
  415. }
  416. partitionParams := []map[string]interface{}{
  417. {
  418. "name": "partition",
  419. "in": "path",
  420. "description": "Partition to select.",
  421. "required": true,
  422. "type": "string",
  423. },
  424. }
  425. binaryData := []map[string]interface{}{
  426. {
  427. "name": "data",
  428. "in": "body",
  429. "description": "The data to store.",
  430. "required": true,
  431. "schema": map[string]interface{}{
  432. "description": "A blob of binary data.",
  433. },
  434. },
  435. }
  436. s["paths"].(map[string]interface{})["/v1/blob/{partition}"] = map[string]interface{}{
  437. "post": map[string]interface{}{
  438. "summary": "Create a binary blob of data.",
  439. "description": "The blob endpoint can be used to store binary data. Its location will be automatically allocated.",
  440. "consumes": []string{
  441. "application/octet-stream",
  442. },
  443. "produces": []string{
  444. "text/plain",
  445. "application/json",
  446. },
  447. "parameters": append(binaryData, partitionParams...),
  448. "responses": map[string]interface{}{
  449. "200": map[string]interface{}{
  450. "description": "The operation was successful.",
  451. "schema": map[string]interface{}{
  452. "type": "object",
  453. "properties": map[string]interface{}{
  454. "id": map[string]interface{}{
  455. "description": "The data ID which can be used to lookup the data.",
  456. "type": "number",
  457. },
  458. },
  459. },
  460. },
  461. "default": map[string]interface{}{
  462. "description": "Error response",
  463. "schema": map[string]interface{}{
  464. "$ref": "#/definitions/Error",
  465. },
  466. },
  467. },
  468. },
  469. }
  470. s["paths"].(map[string]interface{})["/v1/blob/{partition}/{id}"] = map[string]interface{}{
  471. "get": map[string]interface{}{
  472. "summary": "Retrieve a binary blob of data.",
  473. "description": "The blob endpoint can be used to retrieve binary data from a specific location.",
  474. "produces": []string{
  475. "text/plain",
  476. "application/octet-stream",
  477. },
  478. "parameters": append(idParams, partitionParams...),
  479. "responses": map[string]interface{}{
  480. "200": map[string]interface{}{
  481. "description": "The requested binary blob.",
  482. },
  483. "default": map[string]interface{}{
  484. "description": "Error response",
  485. "schema": map[string]interface{}{
  486. "$ref": "#/definitions/Error",
  487. },
  488. },
  489. },
  490. },
  491. "put": map[string]interface{}{
  492. "summary": "Update a binary blob of data.",
  493. "description": "The blob endpoint can be used to update binary data at a specific location.",
  494. "produces": []string{
  495. "text/plain",
  496. },
  497. "parameters": append(idParams, partitionParams...),
  498. "responses": map[string]interface{}{
  499. "200": map[string]interface{}{
  500. "description": "The operation was successful.",
  501. },
  502. "default": map[string]interface{}{
  503. "description": "Error response",
  504. "schema": map[string]interface{}{
  505. "$ref": "#/definitions/Error",
  506. },
  507. },
  508. },
  509. },
  510. "delete": map[string]interface{}{
  511. "summary": "Remove a binary blob of data.",
  512. "description": "The blob endpoint can be used to remove binary data from a specific location.",
  513. "produces": []string{
  514. "text/plain",
  515. },
  516. "parameters": append(idParams, partitionParams...),
  517. "responses": map[string]interface{}{
  518. "200": map[string]interface{}{
  519. "description": "The operation was successful.",
  520. },
  521. "default": map[string]interface{}{
  522. "description": "Error response",
  523. "schema": map[string]interface{}{
  524. "$ref": "#/definitions/Error",
  525. },
  526. },
  527. },
  528. },
  529. }
  530. // Add generic error object to definition
  531. s["definitions"].(map[string]interface{})["Error"] = map[string]interface{}{
  532. "description": "A human readable error mesage.",
  533. "type": "string",
  534. }
  535. }