eliasdb/graph/helpers.go

/*
 * EliasDB
 *
 * Copyright 2016 Matthias Ladkau. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
 */

package graph

import (
	"bytes"
	"encoding/binary"
	"encoding/gob"
	"fmt"
	"strings"

	"devt.de/krotik/common/stringutil"
	"devt.de/krotik/eliasdb/graph/data"
	"devt.de/krotik/eliasdb/graph/util"
	"devt.de/krotik/eliasdb/hash"
	"devt.de/krotik/eliasdb/storage"
)

// Helper functions for GraphManager
// =================================

/*
checkPartitionName checks if a given partition name is valid.
*/
func (gm *Manager) checkPartitionName(part string) error {
	if !stringutil.IsAlphaNumeric(part) {
		return &util.GraphError{
			Type:   util.ErrInvalidData,
			Detail: fmt.Sprintf("Partition name %v is not alphanumeric - can only contain [a-zA-Z0-9_]", part),
		}
	}

	return nil
}

/*
checkNode checks if a given node can be written to the datastore.
*/
func (gm *Manager) checkNode(node data.Node) error {
	return gm.checkItemGeneral(node, "Node")
}

/*
checkItemGeneral checks the general properties of a given graph item.
*/
func (gm *Manager) checkItemGeneral(node data.Node, name string) error {
	if node.Key() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: name + " is missing a key value"}
	}

	if node.Kind() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: name + " is missing a kind value"}
	}

	if !stringutil.IsAlphaNumeric(node.Kind()) {
		return &util.GraphError{
			Type:   util.ErrInvalidData,
			Detail: fmt.Sprintf("%v kind %v is not alphanumeric - can only contain [a-zA-Z0-9_]", name, node.Kind()),
		}
	}

	for attr := range node.Data() {
		if attr == "" {
			return &util.GraphError{Type: util.ErrInvalidData, Detail: name + " contains empty string attribute name"}
		}
	}

	return nil
}

/*
checkEdge checks if a given edge can be written to the datastore.
*/
func (gm *Manager) checkEdge(edge data.Edge) error {
	if err := gm.checkItemGeneral(edge, "Edge"); err != nil {
		return err
	}

	if edge.End1Key() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a key value for end1"}
	}

	if edge.End1Kind() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a kind value for end1"}
	}

	if edge.End1Role() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a role value for end1"}
	} else if !stringutil.IsAlphaNumeric(edge.End1Role()) {
		return &util.GraphError{
			Type:   util.ErrInvalidData,
			Detail: fmt.Sprintf("Edge role %v is not alphanumeric - can only contain [a-zA-Z0-9_]", edge.End1Role()),
		}
	}

	if _, ok := edge.Attr(data.EdgeEnd1Cascading).(bool); !ok {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a cascading value for end1"}
	}

	if edge.End2Key() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a key value for end2"}
	}

	if edge.End2Kind() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a kind value for end2"}
	}

	if edge.End2Role() == "" {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a role value for end2"}
	} else if !stringutil.IsAlphaNumeric(edge.End2Role()) {
		return &util.GraphError{
			Type:   util.ErrInvalidData,
			Detail: fmt.Sprintf("Edge role %v is not alphanumeric - can only contain [a-zA-Z0-9_]", edge.End2Role()),
		}
	}

	if _, ok := edge.Attr(data.EdgeEnd2Cascading).(bool); !ok {
		return &util.GraphError{Type: util.ErrInvalidData, Detail: "Edge is missing a cascading value for end2"}
	}

	return nil
}

/*
writeNodeCount writes a new node count for a specific kind to the datastore.
*/
func (gm *Manager) writeNodeCount(kind string, count uint64, flush bool) error {
	numstr := make([]byte, 8)

	binary.LittleEndian.PutUint64(numstr, count)
	gm.gs.MainDB()[MainDBNodeCount+kind] = string(numstr)

	if flush {
		return gm.gs.FlushMain()
	}

	return nil
}

/*
writeEdgeCount writes a new edge count for a specific kind to the datastore.
*/
func (gm *Manager) writeEdgeCount(kind string, count uint64, flush bool) error {
	numstr := make([]byte, 8)

	binary.LittleEndian.PutUint64(numstr, count)
	gm.gs.MainDB()[MainDBEdgeCount+kind] = string(numstr)

	if flush {
		return gm.gs.FlushMain()
	}

	return nil
}

/*
getNodeStorageHTree gets two HTree instances which can be used to store nodes.
This function ensures that depending entries in other datastructures do exist.
*/
func (gm *Manager) getNodeStorageHTree(part string, kind string,
	create bool) (*hash.HTree, *hash.HTree, error) {

	gm.storageMutex.Lock()
	defer gm.storageMutex.Unlock()

	// Check if the partition name is valid

	if err := gm.checkPartitionName(part); err != nil {
		return nil, nil, err
	}

	// Check if the node kind is valid

	if !stringutil.IsAlphaNumeric(kind) {
		return nil, nil, &util.GraphError{
			Type:   util.ErrInvalidData,
			Detail: fmt.Sprintf("Node kind %v is not alphanumeric - can only contain [a-zA-Z0-9_]", kind),
		}
	}

	// Make sure all required lookup maps are there

	if gm.getMainDBMap(MainDBNodeKinds) == nil {
		gm.storeMainDBMap(MainDBNodeKinds, make(map[string]string))
	}

	if gm.getMainDBMap(MainDBParts) == nil {
		gm.storeMainDBMap(MainDBParts, make(map[string]string))
	}

	if gm.getMainDBMap(MainDBNodeAttrs+kind) == nil {
		gm.storeMainDBMap(MainDBNodeAttrs+kind, make(map[string]string))
	}

	if gm.getMainDBMap(MainDBNodeEdges+kind) == nil {
		gm.storeMainDBMap(MainDBNodeEdges+kind, make(map[string]string))
	}

	if _, ok := gm.gs.MainDB()[MainDBNodeCount+kind]; !ok {
		gm.gs.MainDB()[MainDBNodeCount+kind] = string(make([]byte, 8, 8))
	}

	// Return the actual storage

	gs := gm.gs.StorageManager(part+kind+StorageSuffixNodes, create)
	if gs == nil {
		return nil, nil, nil
	}

	attrTree, err := gm.getHTree(gs, RootIDNodeHTree)
	if err != nil {
		return nil, nil, err
	}

	valTree, err := gm.getHTree(gs, RootIDNodeHTreeSecond)
	if err != nil {
		return nil, nil, err
	}

	return attrTree, valTree, nil
}

/*
getEdgeStorageHTree gets a HTree which can be used to store edges. This function ensures that depending
entries in other datastructures do exist.
*/
func (gm *Manager) getEdgeStorageHTree(part string, kind string, create bool) (*hash.HTree, error) {

	gm.storageMutex.Lock()
	defer gm.storageMutex.Unlock()

	// Check if the partition name is valid

	if err := gm.checkPartitionName(part); err != nil {
		return nil, err
	}

	// Check if the edge kind is valid

	if !stringutil.IsAlphaNumeric(kind) {
		return nil, &util.GraphError{
			Type:   util.ErrInvalidData,
			Detail: fmt.Sprintf("Edge kind %v is not alphanumeric - can only contain [a-zA-Z0-9_]", kind),
		}
	}

	// Make sure all required lookup maps are there

	if gm.getMainDBMap(MainDBEdgeKinds) == nil {
		gm.storeMainDBMap(MainDBEdgeKinds, make(map[string]string))
	}

	if gm.getMainDBMap(MainDBEdgeAttrs+kind) == nil {
		gm.storeMainDBMap(MainDBEdgeAttrs+kind, make(map[string]string))
	}

	if _, ok := gm.gs.MainDB()[MainDBEdgeCount+kind]; !ok {
		gm.gs.MainDB()[MainDBEdgeCount+kind] = string(make([]byte, 8, 8))
	}

	// Return the actual storage

	gs := gm.gs.StorageManager(part+kind+StorageSuffixEdges, create)
	if gs == nil {
		return nil, nil
	}

	return gm.getHTree(gs, RootIDNodeHTree)
}

/*
getNodeIndexHTree gets a HTree which can be used to index nodes.
*/
func (gm *Manager) getNodeIndexHTree(part string, kind string, create bool) (*hash.HTree, error) {
	return gm.getIndexHTree(part, kind, create, "Node", StorageSuffixNodesIndex)
}

/*
getEdgeIndexHTree gets a HTree which can be used to index edges.
*/
func (gm *Manager) getEdgeIndexHTree(part string, kind string, create bool) (*hash.HTree, error) {
	return gm.getIndexHTree(part, kind, create, "Edge", StorageSuffixEdgesIndex)
}

/*
getIndexHTree gets a HTree which can be used to index items.
*/
func (gm *Manager) getIndexHTree(part string, kind string, create bool, name string, suffix string) (*hash.HTree, error) {

	gm.storageMutex.Lock()
	defer gm.storageMutex.Unlock()

	// Check if the partition name is valid

	if err := gm.checkPartitionName(part); err != nil {
		return nil, err
	}

	// Check if the kind is valid

	if !stringutil.IsAlphaNumeric(kind) {
		return nil, &util.GraphError{
			Type:   util.ErrInvalidData,
			Detail: fmt.Sprintf("%v kind %v is not alphanumeric - can only contain [a-zA-Z0-9_]", name, kind),
		}
	}

	gs := gm.gs.StorageManager(part+kind+suffix, create)
	if gs == nil {
		return nil, nil
	}

	return gm.getHTree(gs, RootIDNodeHTree)
}

/*
flushNodeStorage flushes a node storage.
*/
func (gm *Manager) flushNodeStorage(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixNodes, false); sm != nil {
		if err := sm.Flush(); err != nil {
			return &util.GraphError{Type: util.ErrFlushing, Detail: err.Error()}
		}
	}
	return nil
}

/*
flushNodeIndex flushes a node index.
*/
func (gm *Manager) flushNodeIndex(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixNodesIndex, false); sm != nil {
		if err := sm.Flush(); err != nil {
			return &util.GraphError{Type: util.ErrFlushing, Detail: err.Error()}
		}
	}
	return nil
}

/*
flushEdgeStorage flushes an edge storage.
*/
func (gm *Manager) flushEdgeStorage(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixEdges, false); sm != nil {
		if err := sm.Flush(); err != nil {
			return &util.GraphError{Type: util.ErrFlushing, Detail: err.Error()}
		}
	}
	return nil
}

/*
flushEdgeIndex flushes an edge index.
*/
func (gm *Manager) flushEdgeIndex(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixEdgesIndex, false); sm != nil {
		if err := sm.Flush(); err != nil {
			return &util.GraphError{Type: util.ErrFlushing, Detail: err.Error()}
		}
	}
	return nil
}

/*
rollbackNodeStorage rollbacks a node storage.
*/
func (gm *Manager) rollbackNodeStorage(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixNodes, false); sm != nil {
		if err := sm.Rollback(); err != nil {
			return &util.GraphError{Type: util.ErrRollback, Detail: err.Error()}
		}
	}
	return nil
}

/*
rollbackNodeIndex rollbacks a node index.
*/
func (gm *Manager) rollbackNodeIndex(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixNodesIndex, false); sm != nil {
		if err := sm.Rollback(); err != nil {
			return &util.GraphError{Type: util.ErrRollback, Detail: err.Error()}
		}
	}
	return nil
}

/*
rollbackEdgeStorage rollbacks an edge storage.
*/
func (gm *Manager) rollbackEdgeStorage(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixEdges, false); sm != nil {
		if err := sm.Rollback(); err != nil {
			return &util.GraphError{Type: util.ErrRollback, Detail: err.Error()}
		}
	}
	return nil
}

/*
rollbackEdgeIndex rollbacks an edge index.
*/
func (gm *Manager) rollbackEdgeIndex(part string, kind string) error {
	if sm := gm.gs.StorageManager(part+kind+StorageSuffixEdgesIndex, false); sm != nil {
		if err := sm.Rollback(); err != nil {
			return &util.GraphError{Type: util.ErrRollback, Detail: err.Error()}
		}
	}
	return nil
}

/*
getHTree creates or loads a HTree from a given StorageManager. HTrees are not cached
since the creation shouldn't have too much overhead.
*/
func (gm *Manager) getHTree(sm storage.Manager, slot int) (*hash.HTree, error) {
	var htree *hash.HTree
	var err error

	loc := sm.Root(slot)

	if loc == 0 {

		// Create a new HTree and store its location

		htree, err = hash.NewHTree(sm)

		if err != nil {
			err = &util.GraphError{Type: util.ErrAccessComponent, Detail: err.Error()}
		} else {
			sm.SetRoot(slot, htree.Location())
		}

	} else {

		// Load existing HTree

		htree, err = hash.LoadHTree(sm, loc)
		if err != nil {
			err = &util.GraphError{Type: util.ErrAccessComponent, Detail: err.Error()}
		}
	}

	return htree, err
}

/*
getMainDBMap gets a map from the main database.
*/
func (gm *Manager) getMainDBMap(key string) map[string]string {

	// First try to cache

	mapval, ok := gm.mapCache[key]
	if ok {
		return mapval
	}

	// Lookup map and decode it

	val, ok := gm.gs.MainDB()[key]
	if ok {
		mapval = stringToMap(val)
		gm.mapCache[key] = mapval
	}

	return mapval
}

/*
storeMainDBMap stores a map in the main database. The map is stored as a gob byte slice.
Once it has been decoded it is cached for read operations.
*/
func (gm *Manager) storeMainDBMap(key string, mapval map[string]string) {
	gm.mapCache[key] = mapval
	gm.gs.MainDB()[key] = mapToString(mapval)
}

// Static helper functions
// =======================

/*
IsFullSpec is a function to determine if a given spec is a fully specified spec
(i.e. all spec components are specified)
*/
func IsFullSpec(spec string) bool {
	sspec := strings.Split(spec, ":")

	if len(sspec) != 4 || sspec[0] == "" || sspec[1] == "" || sspec[2] == "" || sspec[3] == "" {
		return false
	}

	return true
}

/*
mapToString turns a map of strings into a single string.
*/
func mapToString(stringmap map[string]string) string {
	bb := &bytes.Buffer{}

	gob.NewEncoder(bb).Encode(stringmap)

	return string(bb.Bytes())
}

/*
stringToMap turns a string into a map of strings.
*/
func stringToMap(mapString string) map[string]string {
	var stringmap map[string]string

	if err := gob.NewDecoder(bytes.NewBufferString(mapString)).Decode(&stringmap); err != nil {
		panic(fmt.Sprint("Cannot decode:", mapString, err))
	}

	return stringmap
}