fix: Adding mining example

.gitignore

@@ -1,12 +1,12 @@
-.cache
-.cover
-.eliasdb_console_history
-coverage.txt
-coverage.out
-coverage.html
-test
+/.cache
+/.cover
+/.eliasdb_console_history
+/coverage.txt
+/coverage.out
+/coverage.html
+/test
 /dist
-build
-eliasdb
-examples/tutorial/run/
-examples/chat/run/
+/build
+/eliasdb
+/examples/tutorial/run/
+/examples/chat/run/

README.md

@@ -66,6 +66,10 @@ There is a separate [tutorial](https://devt.de/krotik/eliasdb/src/master/example
 
 The terminal uses a REST API to communicate with the backend. The REST API can be browsed using a dynamically generated swagger.json definition (https://localhost:9090/db/swagger.json). You can browse the API of EliasDB's latest version [here](http://petstore.swagger.io/?url=https://devt.de/krotik/eliasdb/raw/master/swagger.json).
 
+### Clustering:
+
+EliasDB supports to be run in a cluster by joining multiple instances of EliasDB together. You can read more about it [here](https://devt.de/krotik/eliasdb/src/master/cluster.md).
+
 ### Command line options
 The main EliasDB executable has two main tools:
 ```
@@ -219,6 +223,7 @@ Further Reading
 ---------------
 - A design document which describes the different components of the graph database. [Link](https://devt.de/krotik/eliasdb/src/master/eliasdb_design.md)
 - A reference for the EliasDB query language EQL. [Link](https://devt.de/krotik/eliasdb/src/master/eql.md)
+- A reference for the EliasDB's support for GraphQL. [Link](https://devt.de/krotik/eliasdb/src/master/graphql.md)
 - A quick overview of what you can do when you embed EliasDB in your own Go project. [Link](https://devt.de/krotik/eliasdb/src/master/embedding.md)
 
 License

api/v1/cluster_test.go

@@ -424,6 +424,150 @@ func TestClusterQuery(t *testing.T) {
 	}
 }
 
+func TestClusterQueryBigCluster(t *testing.T) {
+	queryURL := "http://localhost" + TESTPORT + EndpointClusterQuery
+
+	// Create a big cluster
+
+	cluster3 := createCluster(3)
+
+	for _, dd := range cluster3 {
+		dd.Start()
+		defer dd.Close()
+	}
+
+	oldGM := api.GM
+	oldGS := api.GS
+	api.GS = cluster3[0]
+	api.GM = graph.NewGraphManager(cluster3[0])
+	api.DD = cluster3[0]
+	api.DDLog = datautil.NewRingBuffer(10)
+
+	defer func() {
+		api.GM = oldGM
+		api.GS = oldGS
+		api.DD = nil
+		api.DDLog = nil
+	}()
+
+	// We should now get back a state
+
+	st, _, res := sendTestRequest(queryURL, "GET", nil)
+
+	if st != "200 OK" || res != `
+{
+  "failed": null,
+  "members": [
+    "TestClusterMember-0",
+    "localhost:9020"
+  ],
+  "replication": 1,
+  "ts": [
+    "TestClusterMember-0",
+    "1"
+  ],
+  "tsold": [
+    "",
+    "0"
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Join more cluster members
+
+	api.DD = cluster3[1]
+	api.DDLog = datautil.NewRingBuffer(10)
+
+	jsonString, err := json.Marshal(map[string]interface{}{
+		"name":    cluster3[0].MemberManager.Name(),
+		"netaddr": cluster3[0].MemberManager.NetAddr(),
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"join", "PUT", jsonString)
+
+	if st != "200 OK" || res != "" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL, "GET", nil)
+
+	if st != "200 OK" || res != `
+{
+  "failed": null,
+  "members": [
+    "TestClusterMember-1",
+    "localhost:9021",
+    "TestClusterMember-0",
+    "localhost:9020"
+  ],
+  "replication": 1,
+  "ts": [
+    "TestClusterMember-0",
+    "2"
+  ],
+  "tsold": [
+    "TestClusterMember-0",
+    "1"
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	api.DD = cluster3[2]
+	api.DDLog = datautil.NewRingBuffer(10)
+
+	jsonString, err = json.Marshal(map[string]interface{}{
+		"name":    cluster3[0].MemberManager.Name(),
+		"netaddr": cluster3[0].MemberManager.NetAddr(),
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"join", "PUT", jsonString)
+
+	if st != "200 OK" || res != "" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL, "GET", nil)
+
+	if st != "200 OK" || res != `
+{
+  "failed": null,
+  "members": [
+    "TestClusterMember-2",
+    "localhost:9022",
+    "TestClusterMember-0",
+    "localhost:9020",
+    "TestClusterMember-1",
+    "localhost:9021"
+  ],
+  "replication": 1,
+  "ts": [
+    "TestClusterMember-0",
+    "3"
+  ],
+  "tsold": [
+    "TestClusterMember-0",
+    "2"
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+}
+
 /*
 Create a cluster with n members (all storage is in memory)
 */

examples/data-mining/build.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+export ROOT_PATH=`pwd`
+
+cd ../..
+docker build --tag data-mining/eliasdb .
+cd $ROOT_PATH
+
+pwd
+
+#if ! [ -d "run" ]; then
+#  mkdir -p run/web
+#  cp -fR res/chat/* run/web
+#  cp -fR res/eliasdb.config.json run
+#  cp -fR res/access.db run
+#fi
+#cd run
+#../../../eliasdb server

examples/data-mining/doc/data-mining.md

@@ -0,0 +1,26 @@
+EliasDB Data Mining Example
+==
+This example demonstrates a more complex application which uses the cluster feature and EQL and GraphQL for data queries.
+
+The idea of the application is to provide a platform for datamining with 3 components for presentation, collection and storage of data.
+
+The tutorial assumes you have downloaded EliasDB, extracted and build it.
+It also assumes that you have a running docker environment with the `docker` and `docker-compose` commands being available. This tutorial will only work in unix-like environments.
+
+For this tutorial please execute "build.sh" in the subdirectory: examples/data-mining.
+
+
+
+
+After starting EliasDB point your browser to:
+```
+https://localhost:9090
+```
+
+The generated default key and certificate for https are self-signed which should give a security warning in the browser. After accepting you should see a login prompt. Enter the credentials for the default user elias:
+```
+Username: elias
+Password: elias
+```
+
+The browser should display the chat application after clicking `Login`. Open a second window and write some chat messages. You can see that both windows update immediately. This is done with GraphQL subscriptions.

examples/data-mining/docker-images/collector/Dockerfile

@@ -0,0 +1,13 @@
+FROM python:alpine
+RUN apk update && apk add --no-cache supervisor
+
+RUN pip install schedule request
+
+ADD etc/supervisord.conf /etc/supervisord.conf
+ADD app/main.py /app/main.py
+
+RUN adduser -D collector
+RUN chown collector:collector /app -R
+USER collector
+
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"]

examples/data-mining/docker-images/collector/app/main.py

@@ -0,0 +1,20 @@
+import schedule
+import time
+import requests
+
+def job():
+    global counter
+
+    now = int(time.time())
+    print("Running requests - timestamp: %s (%s)" %
+        (now, time.strftime("%d-%m-%Y %H:%M:%S", time.gmtime(now))))
+
+    r = requests.get('https://devt.de')
+    res_time = r.elapsed
+    print ("  https://devt.de -> ", res_time)
+
+schedule.every(2).seconds.do(job)
+
+while True:
+    schedule.run_pending()
+    time.sleep(1)

examples/data-mining/docker-images/collector/build.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# Build the collector component
+
+docker build --tag data-mining/collector .
+
+# Run an interactive shell on the build image with:
+# docker run -it data-mining/collector sh

examples/data-mining/docker-images/collector/etc/supervisord.conf

@@ -0,0 +1,11 @@
+[supervisord]
+nodaemon=true
+logfile=/dev/null
+pidfile=/dev/null
+logfile_maxbytes=0
+
+[program:collector]
+command=python -u /app/main.py
+stdout_logfile=/dev/fd/1
+stdout_logfile_maxbytes=0
+redirect_stderr=true

examples/data-mining/docker-images/eliasdb/Dockerfile

@@ -0,0 +1,14 @@
+FROM alpine:latest
+
+ARG cluster_id=""
+
+RUN echo "Building cluster member ${cluster_id}"
+
+COPY ./eliasdb /eliasdb
+COPY ./eliasdb.config.json /data/eliasdb.config.json
+COPY ./cluster.config.json.${cluster_id} /data/cluster.config.json
+COPY ./cluster.stateinfo.${cluster_id} /data/cluster.stateinfo
+
+WORKDIR /data
+
+CMD ["../eliasdb", "server"]

examples/data-mining/docker-images/eliasdb/build.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+export ROOT_PATH=`pwd`
+
+# Build the collector component
+
+cp ../../../../eliasdb .
+docker build --build-arg cluster_id=1 --tag data-mining/eliasdb1 .
+docker build --build-arg cluster_id=2 --tag data-mining/eliasdb2 .
+docker build --build-arg cluster_id=3 --tag data-mining/eliasdb3 .
+
+# Run an interactive shell on the build image with:
+# docker run -it data-mining/eliasdb sh

examples/data-mining/docker-images/eliasdb/cluster.config.json.1

@@ -0,0 +1,6 @@
+{
+    "ClusterMemberName": "eliasdb1",
+    "ClusterMemberRPC": "eliasdb1:9030",
+    "ClusterSecret": "secret123",
+    "ReplicationFactor": 2
+}

examples/data-mining/docker-images/eliasdb/cluster.config.json.2

@@ -0,0 +1,6 @@
+{
+    "ClusterMemberName": "eliasdb2",
+    "ClusterMemberRPC": "eliasdb2:9030",
+    "ClusterSecret": "secret123",
+    "ReplicationFactor": 2
+}

examples/data-mining/docker-images/eliasdb/cluster.config.json.3

@@ -0,0 +1,6 @@
+{
+    "ClusterMemberName": "eliasdb3",
+    "ClusterMemberRPC": "eliasdb3:9030",
+    "ClusterSecret": "secret123",
+    "ReplicationFactor": 2
+}

examples/data-mining/docker-images/eliasdb/docker-compose.yml

@@ -0,0 +1,29 @@
+version: "3"
+
+services:
+  eliasdb1:
+    image: data-mining/eliasdb1:latest
+    container_name: eliasdb1
+    ports:
+      - 4041:9090
+    networks:
+      - back-tier
+
+  eliasdb2:
+    image: data-mining/eliasdb2:latest
+    container_name: eliasdb2
+    ports:
+      - 4042:9090
+    networks:
+      - back-tier
+
+  eliasdb3:
+    image: data-mining/eliasdb3:latest
+    container_name: eliasdb3
+    ports:
+      - 4043:9090
+    networks:
+      - back-tier
+
+networks:
+  back-tier:

examples/data-mining/docker-images/eliasdb/eliasdb.config.json

@@ -0,0 +1,25 @@
+{
+    "ClusterConfigFile": "cluster.config.json",
+    "ClusterLogHistory": 100,
+    "ClusterStateInfoFile": "cluster.stateinfo",
+    "CookieMaxAgeSeconds": "86400",
+    "EnableAccessControl": false,
+    "EnableCluster": true,
+    "EnableClusterTerminal": true,
+    "EnableReadOnly": false,
+    "EnableWebFolder": true,
+    "EnableWebTerminal": true,
+    "HTTPSCertificate": "cert.pem",
+    "HTTPSHost": "127.0.0.1",
+    "HTTPSKey": "key.pem",
+    "HTTPSPort": "9090",
+    "LocationAccessDB": "access.db",
+    "LocationDatastore": "db",
+    "LocationHTTPS": "ssl",
+    "LocationUserDB": "users.db",
+    "LocationWebFolder": "web",
+    "LockFile": "eliasdb.lck",
+    "MemoryOnlyStorage": false,
+    "ResultCacheMaxAgeSeconds": 0,
+    "ResultCacheMaxSize": 0
+}

server/pages.go

@@ -806,7 +806,7 @@ const ClusterTermSRC = `
             padding: 10px;
             border: #000000 2px solid;
             border-radius: 10px;
-            margin: 3em 0 0 0;
+            margin: 3em 0 0 1em;
         }
 
         .c-term h2 {
@@ -821,7 +821,7 @@ const ClusterTermSRC = `
 
         .c-term input {
             padding: 2px;
-            width:100%;
+            width:95%;
         }
 
         .c-term .term_result {
@@ -853,7 +853,7 @@ const ClusterTermSRC = `
             padding: 10px;
             border: #000000 2px solid;
             border-radius: 10px;
-            margin: 3em 0 0 0;
+            margin: 3em 1em 0 0;
         }
 
         .c-log pre {
@@ -874,7 +874,7 @@ const ClusterTermSRC = `
             padding: 10px;
             border: #000000 2px solid;
             border-radius: 10px;
-            margin: 3em 0 0 0;
+            margin: 3em 0 0 1em;
             max-width: 80%;
             word-wrap: break-word;
         }