/* * EliasDB * * Copyright 2016 Matthias Ladkau. All rights reserved. * * This Source Code Form is subject to the terms of the Mozilla Public * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ package interpreter import ( "errors" "fmt" "strings" "testing" "devt.de/krotik/eliasdb/eql/parser" "devt.de/krotik/eliasdb/graph" "devt.de/krotik/eliasdb/graph/data" "devt.de/krotik/eliasdb/graph/graphstorage" "devt.de/krotik/eliasdb/graph/util" "devt.de/krotik/eliasdb/hash" "devt.de/krotik/eliasdb/storage" ) type testNodeInfo struct { *defaultNodeInfo } func (ni *testNodeInfo) SummaryAttributes(kind string) []string { return []string{"key"} } /* Helper function to run a search and check against a result. */ func runSearch(query string, expectedResult string, rt parser.RuntimeProvider) error { ast, err := parser.ParseWithRuntime("test", query, rt) if err != nil { return err } res, err := ast.Runtime.Eval() if err != nil { return err } res.(*SearchResult).StableSort() if fmt.Sprint(res) != expectedResult { return errors.New(fmt.Sprint("Unexpected search result:", res, err)) } return nil } func TestLookup(t *testing.T) { gm, _ := simpleGraph() rt := NewLookupRuntimeProvider("test", "main", gm, NewDefaultNodeInfo(gm)) if err := runSearch("lookup mynode '000', '123'", ` Labels: Mynode Key, Name Format: auto, auto Data: 1:n:key, 1:n:Name 000, Node0 123, Node1 `[1:], rt); err != nil { t.Error(err) return } if err := runSearch("lookup mynode '000', '123' where true", ` Labels: Mynode Key, Name Format: auto, auto Data: 1:n:key, 1:n:Name 000, Node0 123, Node1 `[1:], rt); err != nil { t.Error(err) return } } func TestMultiKindTraversal(t *testing.T) { gm := multiKindGraph() rt := NewGetRuntimeProvider("test", "main", gm, NewDefaultNodeInfo(gm)) // Test simple traversal and setting of primary kind if err := runSearch("get mynode0 traverse :::", ` Labels: Mynode0 Key, Name, Type, Key, Kind, Name Format: auto, auto, auto, auto, auto, auto Data: 1:n:key, 1:n:Name, 1:n:Type, 2:n:key, 2:n:kind, 2:n:name 000, Node0, root, 123, mynode1, 000, Node0, root, 456, mynode2, `[1:], rt); err != nil { t.Error(err) return } // Test simple traversal and access the edge if err := runSearch("get mynode0 traverse :myedge:: end show mynode0:key, 2:n:key, 2:e:key, myedge:name", ` Labels: Mynode0 Key, Key, Key, Myedge Name Format: auto, auto, auto, auto Data: 1:n:key, 2:n:key, 2:e:key, 2:e:name 000, 123, abc1, edge:abc1 000, 456, abc2, edge:abc2 `[1:], rt); err != nil { t.Error(err) return } // Test primary kind ast, err := parser.ParseWithRuntime("test", "get mynode0 traverse ::: end primary mynode2", rt) if err != nil { t.Error(err) return } res, err := ast.Runtime.Eval() if err != nil { t.Error(err) return } if pk := res.(*SearchResult).ResPrimaryKind; pk != "mynode2" { t.Error("Unexpected primary kind:", pk) return } ast, err = parser.ParseWithRuntime("test", "get mynode0 traverse ::: end primary bla", rt) if err != nil { t.Error(err) return } _, err = ast.Runtime.Eval() if err.Error() != "EQL error in test: Unknown node kind (bla) (Line:1 Pos:38)" { t.Error(err) return } } func TestErrors(t *testing.T) { gm, mgs := simpleGraph() rt := NewGetRuntimeProvider("test", "main", gm, NewDefaultNodeInfo(gm)) // Test simple eval error generalProviderMap[parser.NodeGET] = invalidRuntimeInst if err := runSearch("get mynode", "", rt); err.Error() != "EQL error in test: Invalid construct (get) (Line:1 Pos:1)" { t.Error(err) delete(generalProviderMap, parser.NodeGET) return } invast, err := parser.ParseWithRuntime("test", "get mynode", rt) if err != nil { t.Error(err) return } _, err = invast.Runtime.(*invalidRuntime).CondEval(nil, nil) if err.Error() != "EQL error in test: Invalid construct (get) (Line:1 Pos:1)" { t.Error(err) return } delete(generalProviderMap, parser.NodeGET) // Test validation errors if err := runSearch("get mynode show x traverse :::", "", rt); err.Error() != "EQL error in test: Invalid construct (traversals must be before show clause) (Line:1 Pos:19)" { t.Error(err) return } if err := runSearch("get mynode traverse ::", "", rt); err.Error() != "EQL error in test: Invalid traversal spec (::) (Line:1 Pos:12)" { t.Error(err) return } if err := runSearch("get mynode show mynode0:bla", "", rt); err.Error() != "EQL error in test: Invalid construct (Cannot determine data position for kind: mynode0) (Line:1 Pos:17)" { t.Error(err) return } if err := runSearch("get mynode show 0:n:bla", "", rt); err.Error() != "EQL error in test: Invalid construct (Invalid data index: 0:n:bla (index must be greater than 0)) (Line:1 Pos:17)" { t.Error(err) return } if err := runSearch("get mynode show a:n:bla", "", rt); err.Error() != "EQL error in test: Invalid construct (Invalid data index: a:n:bla (strconv.Atoi: parsing \"a\": invalid syntax)) (Line:1 Pos:17)" { t.Error(err) return } if err := runSearch("get mynode show 1:b:bla", "", rt); err.Error() != "EQL error in test: Invalid construct (Invalid data source 'b' (either n - Node or e - Edge)) (Line:1 Pos:17)" { t.Error(err) return } if err := runSearch("get mynode show 2:n:bla", "", rt); err.Error() != "EQL error in test: Invalid column data spec (Data index out of range: 2) (Line:1 Pos:17)" { t.Error(err) return } // Test datastore errors msm := mgs.StorageManager("main"+"mynode"+graph.StorageSuffixNodes, false).(*storage.MemoryStorageManager) msm.AccessMap[1] = storage.AccessCacheAndFetchError if err := runSearch("get mynode", "", rt); err.Error() != "GraphError: Failed to access graph storage component (Slot not found (mystorage/mainmynode.nodes - Location:1))" { t.Error(err) return } delete(msm.AccessMap, 1) // Test nextStartKey error msm.AccessMap[1] = storage.AccessCacheAndFetchSeriousError if _, err := rt.nextStartKey(); err.Error() != "GraphError: Could not read graph information (Record is already in-use (? - ))" { t.Error(err) return } delete(msm.AccessMap, 1) // Test FetchNodePart error msm.AccessMap[3] = storage.AccessCacheAndFetchError if err := runSearch("get mynode", "", rt); err.Error() != "GraphError: Could not read graph information (Slot not found (mystorage/mainmynode.nodes - Location:3))" { t.Error(err) return } delete(msm.AccessMap, 3) // Test TraverseMulti errors msm = mgs.StorageManager("main"+"mynewnode"+graph.StorageSuffixNodes, false).(*storage.MemoryStorageManager) msm.AccessMap[5] = storage.AccessCacheAndFetchError // Node 3 attribute lookup if err := runSearch("get mynode traverse :::mynewnode traverse :::mynewnode end end", "", rt); err.Error() != "GraphError: Could not read graph information (Slot not found (mystorage/mainmynewnode.nodes - Location:5))" { t.Error(err) return } delete(msm.AccessMap, 5) msm.AccessMap[11] = storage.AccessCacheAndFetchError // Traversal spec error if err := runSearch("get mynode traverse :::mynewnode traverse :::mynewnode end end", "", rt); err.Error() != "GraphError: Could not read graph information (Slot not found (mystorage/mainmynewnode.nodes - Location:11))" { t.Error(err) return } delete(msm.AccessMap, 11) msm = mgs.StorageManager("main"+"myedge"+graph.StorageSuffixEdges, false).(*storage.MemoryStorageManager) msmtree, _ := hash.LoadHTree(msm, msm.Root(graph.RootIDNodeHTree)) nm := util.NewNamesManager(mgs.MainDB()) _, slot, _ := msmtree.GetValueAndLocation([]byte(graph.PrefixNSAttr + "abc2" + nm.Encode32("name", false))) msm.AccessMap[slot] = storage.AccessCacheAndFetchError // Edge attribute if err := runSearch("get mynode traverse :::mynewnode end show 1:n:key, 2:n:key, 2:e:key, 2:e:name", "", rt); strings.HasPrefix(err.Error(), "GraphError: Could not read graph information (Slot not found (mystorage/mainmynewnode.nodes - ") { t.Error(err) return } delete(msm.AccessMap, slot) // Test unknown ast node ast, err := parser.ParseWithRuntime("test", "get mynode show x", rt) if err != nil { t.Error(err) return } unknownASTNode := &parser.ASTNode{} unknownASTNode.Name = "Unknown" unknownASTNode.Token = &parser.LexToken{} ast.Children[1].Children = append(ast.Children[1].Children, unknownASTNode) if err = ast.Runtime.Validate(); err.Error() != "EQL error in test: Invalid construct (Unknown)" { t.Error(err) return } ast, err = parser.ParseWithRuntime("test", "get mynode show x as y", rt) if err != nil { t.Error(err) return } ast.Children[1].Children[0].Children = append(ast.Children[1].Children[0].Children, unknownASTNode) if err = ast.Runtime.Validate(); err.Error() != "EQL error in test: Invalid construct (Unknown)" { t.Error(err) return } ast, err = parser.ParseWithRuntime("test", "get mynode traverse ::: traverse ::: end end", rt) if err != nil { t.Error(err) return } ast.Children[1].Children[1].Children = append(ast.Children[1].Children[1].Children, unknownASTNode) if err = ast.Runtime.Validate(); err.Error() != "EQL error in test: Invalid construct (Unknown)" { t.Error(err) return } ast, err = parser.ParseWithRuntime("test", "get mynode", rt) if err != nil { t.Error(err) return } ast.Children = append(ast.Children, unknownASTNode) if err = ast.Runtime.Validate(); err.Error() != "EQL error in test: Invalid construct (Unknown)" { t.Error(err) return } ast.Children = ast.Children[:1] // Test add row errors if err = ast.Runtime.Validate(); err != nil { t.Error(err) return } ast.Runtime.(*getRuntime).rtp.colData[0] = "uu" res := newSearchResult(ast.Runtime.(*getRuntime).rtp.eqlRuntimeProvider, "") if err = res.addRow([]data.Node{data.NewGraphNode()}, nil); err.Error() != "EQL result error in test: Invalid column data spec (Column data spec must have 3 items: uu)" { t.Error(err) return } ast.Runtime.(*getRuntime).rtp.colData[0] = "0:p:test" res = newSearchResult(ast.Runtime.(*getRuntime).rtp.eqlRuntimeProvider, "") if err = res.addRow([]data.Node{data.NewGraphNode()}, nil); err.Error() != "EQL result error in test: Invalid column data spec (Invalid data source 'p' (either n - Node or e - Edge))" { t.Error(err) return } ast.Runtime.(*getRuntime).rtp.colData[0] = "0:e:test" allowMultiEval = true if _, err = ast.Runtime.Eval(); err.Error() != "EQL result error in test: Invalid column data spec (Invalid data index: 0:e:test)" { t.Error(err) return } allowMultiEval = false ast.Runtime.(*getRuntime).rtp.colData[0] = "1:e:test" i := 0 oldNextStartKey := ast.Runtime.(*getRuntime).rtp.nextStartKey ast.Runtime.(*getRuntime).rtp.nextStartKey = func() (string, error) { i++ if i == 3 { return "", errors.New("testerror") } return "000", nil } allowMultiEval = true if _, err = ast.Runtime.Eval(); err.Error() != "testerror" { t.Error(err) return } allowMultiEval = false ast.Runtime.(*getRuntime).rtp.nextStartKey = oldNextStartKey // Test valid result r, err := ast.Runtime.Eval() if err != nil || fmt.Sprint(r) != ` Labels: Mynode Key, Name Format: auto, auto Data: 1:n:key, 1:n:Name 123, Node1 000, Node0 `[1:] { t.Error(r, err) return } } func TestBasicTraversalAndShow(t *testing.T) { gm, _ := simpleGraph() rt := NewGetRuntimeProvider("test", "main", gm, &testNodeInfo{&defaultNodeInfo{gm}}) // Test unknown initial kind ast, err := parser.ParseWithRuntime("test", "get mymissingnode", rt) if err != nil { t.Error(err) return } if _, err := ast.Runtime.Eval(); err.Error() != "EQL error in test: Unknown node kind (mymissingnode) (Line:1 Pos:5)" { t.Error(err) return } // Test simple query with traversals ast, err = parser.ParseWithRuntime("test", ` get mynode traverse :::mynewnode traverse :::mynewnode end end traverse :::mynewnode end`, rt) if err != nil { t.Error(err) return } // Expected result explanation: // Row 1 - Node 000 is alone so no traversals possible // Row 2, 3, 4, 5 - there are two edges from 123 to 456 so possible traversals are doubled // 456 is also connected to 789 and 789-2 // Node 123 has also a direct connection to xxx // 6 and 7 are duplicated because of the two edges from 123 to 456 // 8 is the last combination of twice traversing from 123 to xxx expectedResult := ` Labels: Mynode Key, Mynewnode Key, Mynewnode Key, Mynewnode Key Format: auto, auto, auto, auto Data: 1:n:key, 2:n:key, 3:n:key, 4:n:key 123, 456, 789-2, 456 123, 456, 789-2, 456 123, 456, 789, 456 123, 456, 789, 456 123, xxx ⌘, 789, 456 123, xxx ⌘, 789, 456 123, xxx ⌘, 789, xxx ⌘ `[1:] res, err := ast.Runtime.Eval() if err != nil { t.Error(err) return } res.(*SearchResult).StableSort() if fmt.Sprint(res) != expectedResult { t.Error("Unexpected search result:", res, err) return } // Test showing empty traversals grt := ast.Runtime.(*getRuntime).rtp ast.Runtime.Validate() allowMultiEval = true grt.eqlRuntimeProvider.allowNilTraversal = true res, err = ast.Runtime.Eval() if err != nil { t.Error(err) return } res.(*SearchResult).StableSort() expectedResult = ` Labels: Mynode Key, Mynewnode Key, Mynewnode Key, Mynewnode Key Format: auto, auto, auto, auto Data: 1:n:key, 2:n:key, 3:n:key, 4:n:key 000, , , 123, 456, 789-2, 456 123, 456, 789-2, 456 123, 456, 789, 456 123, 456, 789, 456 123, xxx ⌘, 789, 456 123, xxx ⌘, 789, 456 123, xxx ⌘, 789, xxx ⌘ `[1:] if fmt.Sprint(res) != expectedResult { t.Error("Unexpected search result:", res, err) return } allowMultiEval = false // Test showing the traversal attributes in reverse order ast, err = parser.ParseWithRuntime("test", ` get mynode traverse :::mynewnode traverse :::mynewnode end end show 3:n:key AS key1, 2:n:key FORMAT keystring1, key as key3 FORMAT x`, rt) if err != nil { t.Error(err) return } // Expected result explanation: // Row 1, 2, 3, 4 - there are two edges from 123 to 456 so possible traversals are doubled // 456 is also connected to 789 and 789-2 // Row 5 - Node 123 has also a direct connection to xxx // Row 6 - Node 000 is alone so no traversals possible expectedResult = ` Labels: key1, Key, key3 Format: auto, keystring1, x Data: 3:n:key, 2:n:key, 1:n:key 789-2, 456, 123 789-2, 456, 123 789, 456, 123 789, 456, 123 789, xxx ⌘, 123 `[1:] res, err = ast.Runtime.Eval() if err != nil { t.Error(err) return } res.(*SearchResult).StableSort() if fmt.Sprint(res) != expectedResult { t.Error("Unexpected search result:", res, err) return } } func simpleGraph() (*graph.Manager, *graphstorage.MemoryGraphStorage) { mgs := graphstorage.NewMemoryGraphStorage("mystorage") gm := graph.NewGraphManager(mgs) constructEdge := func(key string, node1 data.Node, node2 data.Node) data.Edge { edge := data.NewGraphEdge() edge.SetAttr("key", key) edge.SetAttr("kind", "myedge") edge.SetAttr(data.EdgeEnd1Key, node1.Key()) edge.SetAttr(data.EdgeEnd1Kind, node1.Kind()) edge.SetAttr(data.EdgeEnd1Role, "node1") edge.SetAttr(data.EdgeEnd1Cascading, true) edge.SetAttr(data.EdgeEnd2Key, node2.Key()) edge.SetAttr(data.EdgeEnd2Kind, node2.Kind()) edge.SetAttr(data.EdgeEnd2Role, "node2") edge.SetAttr(data.EdgeEnd2Cascading, false) edge.SetAttr(data.NodeName, "Edge1"+key) return edge } node0 := data.NewGraphNode() node0.SetAttr("key", "000") node0.SetAttr("kind", "mynode") node0.SetAttr("Name", "Node0") gm.StoreNode("main", node0) node1 := data.NewGraphNode() node1.SetAttr("key", "123") node1.SetAttr("kind", "mynode") node1.SetAttr("Name", "Node1") gm.StoreNode("main", node1) node2 := data.NewGraphNode() node2.SetAttr("key", "456") node2.SetAttr("kind", "mynewnode") node2.SetAttr("Na me", "Node2") gm.StoreNode("main", node2) node3 := data.NewGraphNode() node3.SetAttr("key", "789") node3.SetAttr("kind", "mynewnode") node3.SetAttr("Name", "Node3") gm.StoreNode("main", node3) node33 := data.NewGraphNode() node33.SetAttr("key", "789-2") node33.SetAttr("kind", "mynewnode") node33.SetAttr("Name", "Node3-2") gm.StoreNode("main", node33) node4 := data.NewGraphNode() node4.SetAttr("key", "xxx \xe2\x8c\x98") node4.SetAttr("kind", "mynewnode") node4.SetAttr("Nam \xe2\x8c\x98 e", "Node4") gm.StoreNode("main", node4) gm.StoreEdge("main", constructEdge("abc2", node1, node2)) gm.StoreEdge("main", constructEdge("abc3", node1, node2)) gm.StoreEdge("main", constructEdge("abc1", node2, node3)) gm.StoreEdge("main", constructEdge("abc4", node2, node33)) gm.StoreEdge("main", constructEdge("abc0", node3, node4)) gm.StoreEdge("main", constructEdge("abc99", node4, node1)) return gm, mgs.(*graphstorage.MemoryGraphStorage) } func multiKindGraph() *graph.Manager { mgs := graphstorage.NewMemoryGraphStorage("mystorage") gm := graph.NewGraphManager(mgs) constructEdge := func(key string, node1 data.Node, node2 data.Node) data.Edge { edge := data.NewGraphEdge() edge.SetAttr("key", key) edge.SetAttr("kind", "myedge") edge.SetAttr(data.EdgeEnd1Key, node1.Key()) edge.SetAttr(data.EdgeEnd1Kind, node1.Kind()) edge.SetAttr(data.EdgeEnd1Role, "src") edge.SetAttr(data.EdgeEnd1Cascading, true) edge.SetAttr(data.EdgeEnd2Key, node2.Key()) edge.SetAttr(data.EdgeEnd2Kind, node2.Kind()) edge.SetAttr(data.EdgeEnd2Role, "dest") edge.SetAttr(data.EdgeEnd2Cascading, false) edge.SetAttr(data.NodeName, "edge:"+key) return edge } node0 := data.NewGraphNode() node0.SetAttr("key", "000") node0.SetAttr("kind", "mynode0") node0.SetAttr("Name", "Node0") node0.SetAttr("Type", "root") gm.StoreNode("main", node0) node1 := data.NewGraphNode() node1.SetAttr("key", "123") node1.SetAttr("kind", "mynode1") gm.StoreNode("main", node1) node2 := data.NewGraphNode() node2.SetAttr("key", "456") node2.SetAttr("kind", "mynode2") gm.StoreNode("main", node2) node3 := data.NewGraphNode() node3.SetAttr("key", "789") node3.SetAttr("kind", "mynode3") gm.StoreNode("main", node3) gm.StoreEdge("main", constructEdge("abc1", node0, node1)) gm.StoreEdge("main", constructEdge("abc2", node0, node2)) gm.StoreEdge("main", constructEdge("abc3", node2, node3)) return gm } func dataNodes() *graph.Manager { mgs := graphstorage.NewMemoryGraphStorage("mystorage") gm := graph.NewGraphManager(mgs) node0 := data.NewGraphNode() node0.SetAttr("key", "000") node0.SetAttr("kind", "mynode") node0.SetAttr("name", "Node0") node0.SetAttr("type", "type1") gm.StoreNode("main", node0) node1 := data.NewGraphNode() node1.SetAttr("key", "123") node1.SetAttr("kind", "mynode") node1.SetAttr("name", "Node1") node1.SetAttr("nested.nest1.nest2.atom1", 1.46) node1.SetAttr("type", "type1") gm.StoreNode("main", node1) node2 := data.NewGraphNode() node2.SetAttr("key", "456") node2.SetAttr("kind", "mynode") node2.SetAttr("name", "Node2") node2.SetAttr("type", "type2") node2.SetAttr("nested", map[string]interface{}{ "nest1": map[string]interface{}{ "nest2": map[string]interface{}{ "atom1": 1.45, }, }, }) gm.StoreNode("main", node2) return gm }