Browse Source

feat: Initial commit

Matthias Ladkau 4 years ago
commit
ab26b0c56a
75 changed files with 14507 additions and 0 deletions
  1. 10 0
      .gitignore
  2. 7 0
      LICENSE
  3. 19 0
      NOTICE
  4. 161 0
      README.md
  5. 68 0
      api/about.go
  6. 21 0
      api/about_test.go
  7. 274 0
      api/rest.go
  8. 335 0
      api/rest_test.go
  9. 139 0
      api/swagger.go
  10. 543 0
      api/v1/admin.go
  11. 283 0
      api/v1/admin_test.go
  12. 177 0
      api/v1/dir.go
  13. 158 0
      api/v1/dir_test.go
  14. 839 0
      api/v1/file.go
  15. 1087 0
      api/v1/file_test.go
  16. 66 0
      api/v1/rest.go
  17. 316 0
      api/v1/rest_test.go
  18. 143 0
      api/v1/zip.go
  19. 131 0
      api/v1/zip_test.go
  20. 7 0
      attach_webzip.sh
  21. 766 0
      branch.go
  22. 385 0
      branch_test.go
  23. 258 0
      cli/client.go
  24. 25 0
      cli/dokan.go
  25. 32 0
      cli/dokan_windows.go
  26. 25 0
      cli/fuse.go
  27. 76 0
      cli/fuse_linux.go
  28. 207 0
      cli/rufs.go
  29. 144 0
      cli/server.go
  30. 219 0
      cli/web.go
  31. 83 0
      config/config.go
  32. 43 0
      config/config_test.go
  33. 72 0
      examples/tutorial/doc/tutorial.md
  34. BIN
      examples/tutorial/doc/webclient_browser.png
  35. BIN
      examples/tutorial/doc/webclient_mappings.png
  36. BIN
      examples/tutorial/doc/webclient_menu.png
  37. 16 0
      examples/tutorial/res/rufs.mapping.json
  38. 7 0
      examples/tutorial/res/rufs.server.json
  39. 1 0
      examples/tutorial/res/share/test1.txt
  40. 9 0
      examples/tutorial/start_server.bat
  41. 10 0
      examples/tutorial/start_server.sh
  42. 2 0
      examples/tutorial/start_term_client.bat
  43. 5 0
      examples/tutorial/start_term_client.sh
  44. 2 0
      examples/tutorial/start_web_client.bat
  45. 5 0
      examples/tutorial/start_web_client.sh
  46. 224 0
      export/fuse.go
  47. 43 0
      export/util.go
  48. 158 0
      fileinfo.go
  49. 53 0
      fileinfo_test.go
  50. 9 0
      go.mod
  51. 9 0
      go.sum
  52. 142 0
      integration/rumble/dir.go
  53. 272 0
      integration/rumble/dir_test.go
  54. 359 0
      node/client.go
  55. 76 0
      node/globals.go
  56. 170 0
      node/node.go
  57. 439 0
      node/node_test.go
  58. 182 0
      node/server.go
  59. BIN
      rufs.debug
  60. 1 0
      swagger.json
  61. 61 0
      term/defs.go
  62. 79 0
      term/dir.go
  63. 230 0
      term/file.go
  64. 560 0
      term/file_test.go
  65. 57 0
      term/help.go
  66. 98 0
      term/mount.go
  67. 53 0
      term/sync.go
  68. 157 0
      term/sync_test.go
  69. 174 0
      term/term.go
  70. 479 0
      term/term_test.go
  71. 1356 0
      tree.go
  72. 138 0
      tree_item.go
  73. 1719 0
      tree_test.go
  74. 33 0
      util.go
  75. BIN
      web.zip

+ 10 - 0
.gitignore

@@ -0,0 +1,10 @@
+.cache
+.cover
+coverage.txt
+coverage.out
+coverage.html
+test
+/dist
+build
+rufs
+examples/tutorial/run/

+ 7 - 0
LICENSE

@@ -0,0 +1,7 @@
+Copyright 2017 Matthias Ladkau
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

+ 19 - 0
NOTICE

@@ -0,0 +1,19 @@
+RUFS - Remote Union File System
+Copyright (c) 2017 Matthias Ladkau
+
+The following components are included in this product:
+
+go-fuse
+https://github.com/hanwen/go-fuse
+Copyright (c) 2010 the Go-FUSE Authors
+Licensed under the MIT License
+
+sys - supplemental Go packages
+https://golang.org/x/sys
+Copyright (c) 2009 The Go Authors
+Licensed under the MIT License
+
+dokan-go
+https://github.com/keybase/dokan-go
+Copyright (c) 2016-2017, Keybase
+Licensed under the BSD3 License

+ 161 - 0
README.md

@@ -0,0 +1,161 @@
+Rufs
+====
+Rufs is a remote union filesystem which aims to provide a lightweight and secure solution for distributed file storage. Rufs uses a client-server system where servers expose branches and clients mount one or several branches into a tree structure. The client can overlay branches providing a union view.
+
+Features
+--------
+- Client-Server model using RPC call over ssl.
+- Single executable for client and server.
+- Communication is secured via a secret token which is never transferred over the network and certificate pinning once a client has connected successfully.
+- Clients can provide a unified view with files from different locations.
+- Default client provides CLI, REST API and a web interface.
+- Branches can be read-only.
+- A read-only version of the file system can be exported via FUSE and mounted.
+
+Getting Started
+---------------
+You can download a precompiled package for Windows (win64) or Linux (amd64) [here](https://devt.de/build_status.html).
+
+The archive contains a single executable which contains the server and client code for Rufs.
+
+### Tutorial:
+
+To get an idea of what EliasDB is about have a look at the [tutorial](/examples/local_fileshare/doc/tutorial.md).
+
+### REST API:
+
+The terminal uses a REST API to communicate with the backend. The REST API can be browsed using a dynamically generated swagger.json definition (https://localhost:9090/fs/swagger.json). You can browse the API of Rufs's latest version [here](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/krotik/rufs/master/doc/swagger.json#/default).
+
+### Command line options
+The main Rufs executable has two main tools:
+```
+Rufs 1.0.0
+
+Usage of ./rufs [tool]
+
+The tools are:
+
+    server    Run as a server
+    client    Run as a client
+
+Use ./rufs [tool] --help for more information about a tool.
+```
+The most important one is server which starts the file server. The server has several options:
+```
+Rufs 1.0.0
+
+Usage of ./rufs server [options]
+
+  -config string
+    	Server configuration file (default "rufs.server.json")
+  -help
+    	Show this help message
+  -secret string
+    	Secret file containing the secret token (default "rufs.secret")
+  -ssl-dir string
+    	Directory containing the ssl key.pem and cert.pem files (default "ssl")
+
+The server will automatically create a default config file and
+default directories if nothing is specified.
+```
+Once the server is started the client tool can be used to interact with the server. The options of the client tool are:
+```
+Rufs 1.0.0
+
+Usage of ./rufs client [mapping file]
+
+  -fuse-mount string
+    	Mount tree as FUSE filesystem at specified path (read-only)
+  -help
+    	Show this help message
+  -secret string
+    	Secret file containing the secret token (default "rufs.secret")
+  -ssl-dir string
+    	Directory containing the ssl key.pem and cert.pem files (default "ssl")
+  -web string
+    	Export the tree through a https interface on the specified host:port
+
+The mapping file assignes remote branches to the local tree.
+The client tries to load rufs.mapping.json if no mapping file is defined.
+It starts empty if no mapping file exists. The mapping file
+should have the following json format:
+
+{
+  "branches" : [
+    {
+      "branch"      : <branch name>,
+      "rpc"         : <rpc interface>,
+      "fingerprint" : <fingerprint>
+    },
+    ...
+  ],
+  "tree" : [
+    {
+      "path"      : <path>,
+      "branch"    : <branch name>,
+      "writeable" : <writable flag>
+    },
+    ...
+  ]
+}
+```
+On the console type 'q' to exit and 'help' to get an overview of available commands:
+```
+Available commands:
+----
+branch [branch name] [rpc] [fingerprint] : List all known branches or add a new branch to the tree
+cat <file>                               : Read and print the contents of a file
+cd [path]                                : Show or change the current directory
+checksum [path] [glob]                   : Show a directory listing and file checksums
+cp <src file/dir> <dst dir>              : Copy a file or directory
+dir [path] [glob]                        : Show a directory listing
+get <src file> [dst local file]          : Retrieve a file and store it locally (in the current directory)
+help [cmd]                               : Show general or command specific help
+mkdir <dir>                              : Create a new direectory
+mount [path] [branch name] [ro]          : List all mount points or add a new mount point to the tree
+ping <branch name> [rpc]                 : Ping a remote branch
+put [src local file] [dst file]          : Read a local file and store it
+refresh                                  : Refreshes all known branches and reconnect if possible.
+ren <file> <newfile>                     : Rename a file or directory
+reset [mounts|brances]                   : Remove all mounts or all mounts and all branches
+rm <file>                                : Delete a file or directory (* all files; ** all files/recursive)
+storeconfig [local file]                 : Store the current tree mapping in a local file
+sync <src dir> <dst dir>                 : Make sure dst has the same files and directories as src
+tree [path] [glob]                       : Show the listing of a directory and its subdirectories
+```
+
+### Configuration
+The Rufs client and server use each their own configuration file and require a shared `rufs.secret` file to be able to talk to each other. The server configuration is called `rufs.server.json`. After starting the server for the first time it should create a default configuration file. Available configurations are:
+
+| Configuration Option | Description |
+| --- | --- |
+| BranchName | Branch name which the server will export. |
+| EnableReadOnly | Export the branch only for read operations. |
+| LocalFolder | Local physical folder which is exported. |
+| RPCHost | RPC host for communication with clients. |
+| RPCPort | RPC port for communication with clients. |
+
+Note: It is not (and will never be) possible to access the REST API via HTTP.
+
+Building Rufs
+----------------
+To build Rufs from source you need to have Go installed (go >= 1.12):
+
+Create a directory, change into it and run:
+```
+git clone https://devt.de/krotik/rufs/ .
+```
+
+You can build Rufs's executable with:
+```
+go build -o rufs ./cli/...
+```
+
+Rufs also has a web interface which should be bundled with the executable. The bundled web interface in `web.zip` can be attached by running (assuming the `rufs` executable is in the same folder as the script):
+```
+./attach_webzip.sh
+```
+
+License
+-------
+Rufs source code is available under the [MIT License](/LICENSE).

+ 68 - 0
api/about.go

@@ -0,0 +1,68 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package api contains the REST API for RUFS.
+
+/about
+
+Endpoint which returns an object with version information.
+
+{
+    api_versions : List of available API versions e.g. [ "v1" ]
+    product      : Name of the API provider (RUFS)
+    version:     : Version of the API provider
+}
+*/
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"devt.de/krotik/rufs/config"
+)
+
+/*
+EndpointAbout is the about endpoint definition (rooted). Handles about/
+*/
+const EndpointAbout = APIRoot + "/about/"
+
+/*
+AboutEndpointInst creates a new endpoint handler.
+*/
+func AboutEndpointInst() RestEndpointHandler {
+	return &aboutEndpoint{}
+}
+
+/*
+aboutEndpoint is the handler object for about operations.
+*/
+type aboutEndpoint struct {
+	*DefaultEndpointHandler
+}
+
+/*
+HandleGET returns about data for the REST API.
+*/
+func (a *aboutEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+
+	data := map[string]interface{}{
+		"product": "RUFS",
+		"version": config.ProductVersion,
+	}
+
+	// Write data
+
+	w.Header().Set("content-type", "application/json; charset=utf-8")
+
+	ret := json.NewEncoder(w)
+	ret.Encode(data)
+}

+ 21 - 0
api/about_test.go

@@ -0,0 +1,21 @@
+package api
+
+import (
+	"testing"
+
+	"devt.de/krotik/rufs/config"
+)
+
+func TestAboutEndpoint(t *testing.T) {
+
+	st, _, body, _ := sendTestRequest(testQueryURL+EndpointAbout, "GET", nil)
+
+	if st != "200 OK" || body != `
+{
+  "product": "RUFS",
+  "version": "`[1:]+config.ProductVersion+`"
+}` {
+		t.Error("Unexpected response:", st, body)
+		return
+	}
+}

+ 274 - 0
api/rest.go

@@ -0,0 +1,274 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package api
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net/http"
+	"strings"
+	"sync"
+
+	"devt.de/krotik/rufs"
+)
+
+/*
+APIVersion is the version of the REST API
+*/
+const APIVersion = "1.0.0"
+
+/*
+APIRoot is the API root directory for the REST API
+*/
+const APIRoot = "/fs"
+
+/*
+APISchemes defines the supported schemes by the REST API
+*/
+var APISchemes = []string{"https"}
+
+/*
+APIHost is the host definition for the REST API
+*/
+var APIHost = "localhost:9040"
+
+/*
+RestEndpointInst models a factory function for REST endpoint handlers.
+*/
+type RestEndpointInst func() RestEndpointHandler
+
+/*
+GeneralEndpointMap is a map of urls to general REST endpoints
+*/
+var GeneralEndpointMap = map[string]RestEndpointInst{
+	EndpointAbout:   AboutEndpointInst,
+	EndpointSwagger: SwaggerEndpointInst,
+}
+
+/*
+RestEndpointHandler models a REST endpoint handler.
+*/
+type RestEndpointHandler interface {
+
+	/*
+		HandleGET handles a GET request.
+	*/
+	HandleGET(w http.ResponseWriter, r *http.Request, resources []string)
+
+	/*
+		HandlePOST handles a POST request.
+	*/
+	HandlePOST(w http.ResponseWriter, r *http.Request, resources []string)
+
+	/*
+		HandlePUT handles a PUT request.
+	*/
+	HandlePUT(w http.ResponseWriter, r *http.Request, resources []string)
+
+	/*
+		HandleDELETE handles a DELETE request.
+	*/
+	HandleDELETE(w http.ResponseWriter, r *http.Request, resources []string)
+
+	/*
+		SwaggerDefs is used to describe the endpoint in swagger.
+	*/
+	SwaggerDefs(s map[string]interface{})
+}
+
+/*
+trees is a map of all trees which can be used by the REST API
+*/
+var trees = make(map[string]*rufs.Tree)
+var treesLock = sync.Mutex{}
+
+/*
+ResetTrees removes all registered trees.
+*/
+var ResetTrees = func() {
+	treesLock.Lock()
+	defer treesLock.Unlock()
+
+	trees = make(map[string]*rufs.Tree)
+}
+
+/*
+Trees is a getter function which returns a map of all registered trees.
+This function can be overwritten by client code to implement access
+control.
+*/
+var Trees = func() (map[string]*rufs.Tree, error) {
+	treesLock.Lock()
+	defer treesLock.Unlock()
+
+	ret := make(map[string]*rufs.Tree)
+
+	for k, v := range trees {
+		ret[k] = v
+	}
+
+	return ret, nil
+}
+
+/*
+GetTree returns a specific tree. This function can be overwritten by
+client code to implement access control.
+*/
+var GetTree = func(id string) (*rufs.Tree, bool, error) {
+	treesLock.Lock()
+	defer treesLock.Unlock()
+
+	tree, ok := trees[id]
+
+	return tree, ok, nil
+}
+
+/*
+AddTree adds a new tree. This function can be overwritten by
+client code to implement access control.
+*/
+var AddTree = func(id string, tree *rufs.Tree) error {
+	treesLock.Lock()
+	defer treesLock.Unlock()
+
+	if _, ok := trees[id]; ok {
+		return fmt.Errorf("Tree %v already exists", id)
+	}
+
+	trees[id] = tree
+
+	return nil
+}
+
+/*
+RemoveTree removes a tree. This function can be overwritten by
+client code to implement access control.
+*/
+var RemoveTree = func(id string) error {
+	treesLock.Lock()
+	defer treesLock.Unlock()
+
+	if _, ok := trees[id]; !ok {
+		return fmt.Errorf("Tree %v does not exist", id)
+	}
+
+	delete(trees, id)
+
+	return nil
+}
+
+/*
+TreeConfigTemplate is the configuration which is used for newly created trees.
+*/
+var TreeConfigTemplate map[string]interface{}
+
+/*
+TreeCertTemplate is the certificate which is used for newly created trees.
+*/
+var TreeCertTemplate *tls.Certificate
+
+/*
+Map of all registered endpoint handlers.
+*/
+var registered = map[string]RestEndpointInst{}
+
+/*
+HandleFunc to use for registering handlers
+*/
+var HandleFunc = http.HandleFunc
+
+/*
+RegisterRestEndpoints registers all given REST endpoint handlers.
+*/
+func RegisterRestEndpoints(endpointInsts map[string]RestEndpointInst) {
+
+	for url, endpointInst := range endpointInsts {
+		registered[url] = endpointInst
+
+		HandleFunc(url, func() func(w http.ResponseWriter, r *http.Request) {
+			var handlerURL = url
+			var handlerInst = endpointInst
+
+			return func(w http.ResponseWriter, r *http.Request) {
+
+				// Create a new handler instance
+
+				handler := handlerInst()
+
+				// Handle request in appropriate method
+
+				res := strings.TrimSpace(r.URL.Path[len(handlerURL):])
+
+				if len(res) > 0 && res[len(res)-1] == '/' {
+					res = res[:len(res)-1]
+				}
+
+				var resources []string
+
+				if res != "" {
+					resources = strings.Split(res, "/")
+				}
+
+				switch r.Method {
+				case "GET":
+					handler.HandleGET(w, r, resources)
+
+				case "POST":
+					handler.HandlePOST(w, r, resources)
+
+				case "PUT":
+					handler.HandlePUT(w, r, resources)
+
+				case "DELETE":
+					handler.HandleDELETE(w, r, resources)
+
+				default:
+					http.Error(w, http.StatusText(http.StatusMethodNotAllowed),
+						http.StatusMethodNotAllowed)
+				}
+			}
+		}())
+	}
+}
+
+/*
+DefaultEndpointHandler is the default endpoint handler implementation.
+*/
+type DefaultEndpointHandler struct {
+}
+
+/*
+HandleGET handles a GET request.
+*/
+func (de *DefaultEndpointHandler) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+}
+
+/*
+HandlePOST handles a POST request.
+*/
+func (de *DefaultEndpointHandler) HandlePOST(w http.ResponseWriter, r *http.Request, resources []string) {
+	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+}
+
+/*
+HandlePUT handles a PUT request.
+*/
+func (de *DefaultEndpointHandler) HandlePUT(w http.ResponseWriter, r *http.Request, resources []string) {
+	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+}
+
+/*
+HandleDELETE handles a DELETE request.
+*/
+func (de *DefaultEndpointHandler) HandleDELETE(w http.ResponseWriter, r *http.Request, resources []string) {
+	http.Error(w, "Method Not Allowed", http.StatusMethodNotAllowed)
+}

+ 335 - 0
api/rest_test.go

@@ -0,0 +1,335 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"strings"
+	"sync"
+	"testing"
+
+	"devt.de/krotik/common/httputil"
+	"devt.de/krotik/rufs"
+)
+
+const testPort = ":9040"
+
+var testQueryURL = "http://localhost" + testPort
+
+var lastRes []string
+
+type testEndpoint struct {
+	*DefaultEndpointHandler
+}
+
+/*
+handleSearchQuery handles a search query REST call.
+*/
+func (te *testEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+	lastRes = resources
+	te.DefaultEndpointHandler.HandleGET(w, r, resources)
+}
+
+func (te *testEndpoint) SwaggerDefs(s map[string]interface{}) {
+}
+
+var testEndpointMap = map[string]RestEndpointInst{
+	"/": func() RestEndpointHandler {
+		return &testEndpoint{}
+	},
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	hs, wg := startServer()
+	if hs == nil {
+		return
+	}
+	defer stopServer(hs, wg)
+
+	RegisterRestEndpoints(testEndpointMap)
+	RegisterRestEndpoints(GeneralEndpointMap)
+
+	// Run the tests
+
+	res := m.Run()
+
+	// Teardown
+
+	stopServer(hs, wg)
+
+	os.Exit(res)
+}
+
+func TestTreeManagement(t *testing.T) {
+
+	if err := AddTree("1", &rufs.Tree{}); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := AddTree("1", &rufs.Tree{}); err == nil || err.Error() != "Tree 1 already exists" {
+		t.Error(err)
+		return
+	}
+
+	if res, _ := Trees(); fmt.Sprint(res) != "map[1:/: ]" {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+
+	if res, _, _ := GetTree("1"); res == nil {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+
+	if err := RemoveTree("1"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := RemoveTree("1"); err == nil || err.Error() != "Tree 1 does not exist" {
+		t.Error(err)
+		return
+	}
+
+	if err := AddTree("1", &rufs.Tree{}); err != nil {
+		t.Error(err)
+		return
+	}
+
+	ResetTrees()
+
+	if res, _ := Trees(); fmt.Sprint(res) != "map[]" {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+}
+
+func TestEndpointHandling(t *testing.T) {
+
+	lastRes = nil
+
+	if _, _, res, _ := sendTestRequest(testQueryURL, "GET", nil); res != "Method Not Allowed" {
+		t.Error("Unexpected response:", res)
+		return
+	}
+
+	if lastRes != nil {
+		t.Error("Unexpected lastRes:", lastRes)
+	}
+
+	lastRes = nil
+
+	if _, _, res, _ := sendTestRequest(testQueryURL+"/foo/bar", "GET", nil); res != "Method Not Allowed" {
+		t.Error("Unexpected response:", res)
+		return
+	}
+
+	if fmt.Sprint(lastRes) != "[foo bar]" {
+		t.Error("Unexpected lastRes:", lastRes)
+	}
+
+	lastRes = nil
+
+	if _, _, res, _ := sendTestRequest(testQueryURL+"/foo/bar/", "GET", nil); res != "Method Not Allowed" {
+		t.Error("Unexpected response:", res)
+		return
+	}
+
+	if fmt.Sprint(lastRes) != "[foo bar]" {
+		t.Error("Unexpected lastRes:", lastRes)
+	}
+
+	if _, _, res, _ := sendTestRequest(testQueryURL, "POST", nil); res != "Method Not Allowed" {
+		t.Error("Unexpected response:", res)
+		return
+	}
+
+	if _, _, res, _ := sendTestRequest(testQueryURL, "PUT", nil); res != "Method Not Allowed" {
+		t.Error("Unexpected response:", res)
+		return
+	}
+
+	if _, _, res, _ := sendTestRequest(testQueryURL, "DELETE", nil); res != "Method Not Allowed" {
+		t.Error("Unexpected response:", res)
+		return
+	}
+
+	if _, _, res, _ := sendTestRequest(testQueryURL, "UPDATE", nil); res != "Method Not Allowed" {
+		t.Error("Unexpected response:", res)
+		return
+	}
+
+	// Test swagger endpoint
+
+	if _, _, res, _ := sendTestRequest(testQueryURL+"/fs/swagger.json", "GET", nil); res != `
+{
+  "basePath": "/fs",
+  "definitions": {
+    "Error": {
+      "description": "A human readable error mesage.",
+      "type": "string"
+    }
+  },
+  "host": "localhost:9040",
+  "info": {
+    "description": "Query and control the Remote Union File System.",
+    "title": "RUFS API",
+    "version": "1.0.0"
+  },
+  "paths": {
+    "/about": {
+      "get": {
+        "description": "Returns available API versions, product name and product version.",
+        "produces": [
+          "application/json"
+        ],
+        "responses": {
+          "200": {
+            "description": "About info object",
+            "schema": {
+              "properties": {
+                "api_versions": {
+                  "description": "List of available API versions.",
+                  "items": {
+                    "description": "Available API version.",
+                    "type": "string"
+                  },
+                  "type": "array"
+                },
+                "product": {
+                  "description": "Product name of the REST API provider.",
+                  "type": "string"
+                },
+                "version": {
+                  "description": "Version of the REST API provider.",
+                  "type": "string"
+                }
+              },
+              "type": "object"
+            }
+          },
+          "default": {
+            "description": "Error response",
+            "schema": {
+              "$ref": "#/definitions/Error"
+            }
+          }
+        },
+        "summary": "Return information about the REST API provider."
+      }
+    }
+  },
+  "produces": [
+    "application/json"
+  ],
+  "schemes": [
+    "https"
+  ],
+  "swagger": "2.0"
+}`[1:] {
+		t.Error("Unexpected response:", res)
+		return
+	}
+}
+
+/*
+Send a request to a HTTP test server
+*/
+func sendTestRequest(url string, method string, content []byte) (string, http.Header, string, *http.Response) {
+	var req *http.Request
+	var err error
+
+	if content != nil {
+		req, err = http.NewRequest(method, url, bytes.NewBuffer(content))
+	} else {
+		req, err = http.NewRequest(method, url, nil)
+	}
+
+	if err != nil {
+		panic(err)
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := ioutil.ReadAll(resp.Body)
+	bodyStr := strings.Trim(string(body), " \n")
+
+	// Try json decoding first
+
+	out := bytes.Buffer{}
+	err = json.Indent(&out, []byte(bodyStr), "", "  ")
+	if err == nil {
+		return resp.Status, resp.Header, out.String(), resp
+	}
+
+	// Just return the body
+
+	return resp.Status, resp.Header, bodyStr, resp
+}
+
+/*
+Start a HTTP test server.
+*/
+func startServer() (*httputil.HTTPServer, *sync.WaitGroup) {
+	hs := &httputil.HTTPServer{}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go hs.RunHTTPServer(testPort, &wg)
+
+	wg.Wait()
+
+	// Server is started
+
+	if hs.LastError != nil {
+		panic(hs.LastError)
+	}
+
+	return hs, &wg
+}
+
+/*
+Stop a started HTTP test server.
+*/
+func stopServer(hs *httputil.HTTPServer, wg *sync.WaitGroup) {
+
+	if hs.Running == true {
+
+		wg.Add(1)
+
+		// Server is shut down
+
+		hs.Shutdown()
+
+		wg.Wait()
+
+	} else {
+
+		panic("Server was not running as expected")
+	}
+}

+ 139 - 0
api/swagger.go

@@ -0,0 +1,139 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+)
+
+/*
+SwaggerDefs is used to describe the endpoint in swagger.
+*/
+func (a *aboutEndpoint) SwaggerDefs(s map[string]interface{}) {
+
+	// Add query paths
+
+	s["paths"].(map[string]interface{})["/about"] = map[string]interface{}{
+		"get": map[string]interface{}{
+			"summary":     "Return information about the REST API provider.",
+			"description": "Returns available API versions, product name and product version.",
+			"produces": []string{
+				"application/json",
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "About info object",
+					"schema": map[string]interface{}{
+						"type": "object",
+						"properties": map[string]interface{}{
+							"api_versions": map[string]interface{}{
+								"description": "List of available API versions.",
+								"type":        "array",
+								"items": map[string]interface{}{
+									"description": "Available API version.",
+									"type":        "string",
+								},
+							},
+							"product": map[string]interface{}{
+								"description": "Product name of the REST API provider.",
+								"type":        "string",
+							},
+							"version": map[string]interface{}{
+								"description": "Version of the REST API provider.",
+								"type":        "string",
+							},
+						},
+					},
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	// Add generic error object to definition
+
+	s["definitions"].(map[string]interface{})["Error"] = map[string]interface{}{
+		"description": "A human readable error mesage.",
+		"type":        "string",
+	}
+}
+
+/*
+EndpointSwagger is the swagger endpoint URL (rooted). Handles swagger.json/
+*/
+const EndpointSwagger = APIRoot + "/swagger.json/"
+
+/*
+SwaggerEndpointInst creates a new endpoint handler.
+*/
+func SwaggerEndpointInst() RestEndpointHandler {
+	return &swaggerEndpoint{}
+}
+
+/*
+Handler object for swagger operations.
+*/
+type swaggerEndpoint struct {
+	*DefaultEndpointHandler
+}
+
+/*
+HandleGET returns the swagger definition of the REST API.
+*/
+func (a *swaggerEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+
+	// Add general sections
+
+	data := map[string]interface{}{
+		"swagger":     "2.0",
+		"host":        APIHost,
+		"schemes":     APISchemes,
+		"basePath":    APIRoot,
+		"produces":    []string{"application/json"},
+		"paths":       map[string]interface{}{},
+		"definitions": map[string]interface{}{},
+	}
+
+	// Go through all registered components and let them add their definitions
+
+	a.SwaggerDefs(data)
+
+	for _, inst := range registered {
+		inst().SwaggerDefs(data)
+	}
+
+	// Write data
+
+	w.Header().Set("content-type", "application/json; charset=utf-8")
+
+	ret := json.NewEncoder(w)
+	ret.Encode(data)
+}
+
+/*
+SwaggerDefs is used to describe the endpoint in swagger.
+*/
+func (a *swaggerEndpoint) SwaggerDefs(s map[string]interface{}) {
+
+	// Add general application information
+
+	s["info"] = map[string]interface{}{
+		"title":       "RUFS API",
+		"description": "Query and control the Remote Union File System.",
+		"version":     APIVersion,
+	}
+}

+ 543 - 0
api/v1/admin.go

@@ -0,0 +1,543 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package v1 contains Rufs REST API Version 1.
+
+Admin control endpoint
+
+/admin
+
+The admin endpoint can be used for various admin tasks such as registering
+new branches or mounting known branches.
+
+A GET request to the admin endpoint returns the current tree
+configuration; an object of all known branches and the current mapping:
+
+	{
+	    branches : [ <known branches> ],
+	    tree  : [ <current mapping> ]
+	}
+
+A POST request to the admin endpoint creates a new tree. The body of
+the request should have the following form:
+
+	"<name>"
+
+/admin/<tree>
+
+A DELETE request to a particular tree will delete the tree.
+
+/admin/<tree>/branch
+
+A new branch can be created in an existing tree by sending a POST request
+to the branch endpoint. The body of the request should have the following
+form:
+
+	{
+	    branch : <Name of the branch>,
+	    rpc : <RPC definition of the remote branch (e.g. localhost:9020)>,
+	    fingerprint : <Expected SSL fingerprint of the remote branch or an empty string>
+	}
+
+/admin/<tree>/mapping
+
+A new mapping can be created in an existing tree by sending a POST request
+to the mapping endpoint. The body of the request should have the following
+form:
+
+	{
+	    branch : <Name of the branch>,
+	    dir : <Tree directory of the branch root>,
+	    writable : <Flag if the branch should handle write operations>
+	}
+
+
+Dir listing endpoing
+
+/dir/<tree>/<path>
+
+The dir endpoing handles requests for the directory listing of a certain
+path. A request url should be of the following form:
+
+/dir/<tree>/<path>?recursive=<flag>&checksums=<flag>
+
+The request can optionally include the flag parameters (value should
+be 1 or 0) recursive and checksums. The recursive flag will add all
+subdirectories to the listing and the checksums flag will add
+checksums for all listed files.
+
+
+File queries and manipulation
+
+/file/{tree}/{path}
+
+A GET request to a specific file will return its contents. A POST will
+upload a new or overwrite an existing file. A DELETE request will delete
+an existing file.
+
+New files are expected to be uploaded using a multipart/form-data request.
+When uploading a new file the form field for the file should be named
+"uploadfile". The form can optionally contain a redirect field which
+will issue a redirect once the file has been uploaded.
+
+A PUT request is used to perform a file operation. The request body
+should be a JSON object of the form (parameters are operation specific):
+
+	{
+	    action : <Action to perform>,
+		files : <List of (full path) files which should be copied / renamed>
+	    newname : <New name of file (when renaming)>,
+		newnames : <List of new file names when renaming multiple files using
+					the files parameter>,
+	    destination : <Destination file when copying a single file - Destination
+						directory when copying multiple files using the files
+						parameter or syncing directories>
+	}
+
+The action can either be: sync, rename, mkdir or copy. Copy and sync returns a JSON
+structure containing a progress id:
+
+	{
+	    progress_id : <Id for progress of the copy operation>
+	}
+
+
+Progress information
+
+/progress/<progress id>
+
+A GET request to the progress endpoint returns the current progress of
+an ongoing operation. The result should be:
+
+	{
+	    "item": <Currently processing item>,
+	    "operation": <Name of operation>,
+	    "progress": <Current progress>,
+	    "subject": <Name of the subject on which the operation is performed>,
+	    "total_items": <Total number of items>,
+	    "total_progress": <Total progress>
+	}
+
+
+Create zip files
+
+/zip/<tree>
+
+A post to the zip enpoint returns a zip file containing requested files. The
+files to include must be given as a list of file name with full path in the body.
+The body should be application/x-www-form-urlencoded encoded. The list should
+be a JSON encoded string as value of the value files. The body should have the
+following form:
+
+	files=[ "<file1>", "<file2>" ]
+*/
+package v1
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+)
+
+/*
+EndpointAdmin is the mount endpoint URL (rooted). Handles everything
+under admin/...
+*/
+const EndpointAdmin = api.APIRoot + APIv1 + "/admin/"
+
+/*
+AdminEndpointInst creates a new endpoint handler.
+*/
+func AdminEndpointInst() api.RestEndpointHandler {
+	return &adminEndpoint{}
+}
+
+/*
+Handler object for admin operations.
+*/
+type adminEndpoint struct {
+	*api.DefaultEndpointHandler
+}
+
+/*
+HandleGET handles an admin query REST call.
+*/
+func (a *adminEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+	data := make(map[string]interface{})
+
+	trees, err := api.Trees()
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	refreshName := r.URL.Query().Get("refresh")
+
+	for k, v := range trees {
+		var tree map[string]interface{}
+
+		if refreshName != "" && k == refreshName {
+			v.Refresh()
+		}
+
+		json.Unmarshal([]byte(v.Config()), &tree)
+		data[k] = tree
+	}
+
+	// Write data
+
+	w.Header().Set("content-type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(data)
+}
+
+/*
+HandlePOST handles REST calls to create a new tree.
+*/
+func (a *adminEndpoint) HandlePOST(w http.ResponseWriter, r *http.Request, resources []string) {
+	var tree *rufs.Tree
+	var ok bool
+	var err error
+	var data map[string]interface{}
+
+	if len(resources) == 0 {
+		var name string
+
+		if err := json.NewDecoder(r.Body).Decode(&name); err != nil {
+			http.Error(w, fmt.Sprintf("Could not decode request body: %v", err.Error()),
+				http.StatusBadRequest)
+			return
+		} else if name == "" {
+			http.Error(w, fmt.Sprintf("Body must contain the tree name as a non-empty JSON string"),
+				http.StatusBadRequest)
+			return
+		}
+
+		// Create a new tree
+
+		tree, err := rufs.NewTree(api.TreeConfigTemplate, api.TreeCertTemplate)
+		if err != nil {
+			http.Error(w, fmt.Sprintf("Could not create new tree: %v", err.Error()),
+				http.StatusBadRequest)
+			return
+		}
+
+		// Store the new tree
+
+		if err := api.AddTree(name, tree); err != nil {
+			http.Error(w, fmt.Sprintf("Could not add new tree: %v", err.Error()),
+				http.StatusBadRequest)
+		}
+
+		return
+	}
+
+	if !checkResources(w, resources, 2, 2, "Need a tree name and a section (either branches or mapping)") {
+		return
+	}
+
+	if tree, ok, err = api.GetTree(resources[0]); err == nil && !ok {
+		err = fmt.Errorf("Unknown tree: %v", resources[0])
+	}
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	if err := json.NewDecoder(r.Body).Decode(&data); err != nil {
+		http.Error(w, fmt.Sprintf("Could not decode request body: %v", err.Error()),
+			http.StatusBadRequest)
+		return
+	}
+
+	if resources[1] == "branch" {
+
+		// Add a new branch
+
+		if rpc, ok := getMapValue(w, data, "rpc"); ok {
+			if branch, ok := getMapValue(w, data, "branch"); ok {
+				if fingerprint, ok := getMapValue(w, data, "fingerprint"); ok {
+
+					if err := tree.AddBranch(branch, rpc, fingerprint); err != nil {
+						http.Error(w, fmt.Sprintf("Could not add branch: %v", err.Error()),
+							http.StatusBadRequest)
+					}
+				}
+			}
+		}
+
+	} else if resources[1] == "mapping" {
+
+		// Add a new mapping
+
+		if _, ok := data["dir"]; ok {
+
+			if dir, ok := getMapValue(w, data, "dir"); ok {
+				if branch, ok := getMapValue(w, data, "branch"); ok {
+					if writeableStr, ok := getMapValue(w, data, "writeable"); ok {
+
+						writeable, err := strconv.ParseBool(writeableStr)
+
+						if err != nil {
+							http.Error(w, fmt.Sprintf("Writeable value must be a boolean: %v", err.Error()),
+								http.StatusBadRequest)
+
+						} else if err := tree.AddMapping(dir, branch, writeable); err != nil {
+
+							http.Error(w, fmt.Sprintf("Could not add branch: %v", err.Error()),
+								http.StatusBadRequest)
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+/*
+HandleDELETE handles REST calls to delete an existing tree.
+*/
+func (a *adminEndpoint) HandleDELETE(w http.ResponseWriter, r *http.Request, resources []string) {
+
+	if !checkResources(w, resources, 1, 1, "Need a tree name") {
+		return
+	}
+
+	// Delete the tree
+
+	if err := api.RemoveTree(resources[0]); err != nil {
+		http.Error(w, fmt.Sprintf("Could not remove tree: %v", err.Error()),
+			http.StatusBadRequest)
+	}
+}
+
+/*
+SwaggerDefs is used to describe the endpoint in swagger.
+*/
+func (a *adminEndpoint) SwaggerDefs(s map[string]interface{}) {
+
+	s["paths"].(map[string]interface{})["/v1/admin"] = map[string]interface{}{
+		"get": map[string]interface{}{
+			"summary":     "Return all current tree configurations.",
+			"description": "All current tree configurations; each object has a list of all known branches and the current mapping.",
+			"produces": []string{
+				"text/plain",
+				"application/json",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "refresh",
+					"in":          "query",
+					"description": "Refresh a particular tree (reload branches and mappings).",
+					"required":    false,
+					"type":        "string",
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "A key-value map of tree name to tree configuration",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+		"post": map[string]interface{}{
+			"summary":     "Create a new tree.",
+			"description": "Create a new named tree.",
+			"consumes": []string{
+				"application/json",
+			},
+			"produces": []string{
+				"text/plain",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "data",
+					"in":          "body",
+					"description": "Name of the new tree.",
+					"required":    true,
+					"schema": map[string]interface{}{
+						"type": "string",
+					},
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns an empty body if successful.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	s["paths"].(map[string]interface{})["/v1/admin/{tree}"] = map[string]interface{}{
+		"delete": map[string]interface{}{
+			"summary":     "Delete a tree.",
+			"description": "Delete a named tree.",
+			"produces": []string{
+				"text/plain",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns an empty body if successful.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	s["paths"].(map[string]interface{})["/v1/admin/{tree}/branch"] = map[string]interface{}{
+		"post": map[string]interface{}{
+			"summary":     "Add a new branch.",
+			"description": "Add a new remote branch to the tree.",
+			"consumes": []string{
+				"application/json",
+			},
+			"produces": []string{
+				"text/plain",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "data",
+					"in":          "body",
+					"description": "Definition of the new branch.",
+					"required":    true,
+					"schema": map[string]interface{}{
+						"type": "object",
+						"properties": map[string]interface{}{
+							"branch": map[string]interface{}{
+								"description": "Name of the remote branch (must match on the remote branch).",
+								"type":        "string",
+							},
+							"rpc": map[string]interface{}{
+								"description": "RPC definition of the remote branch (e.g. localhost:9020).",
+								"type":        "string",
+							},
+							"fingerprint": map[string]interface{}{
+								"description": "Expected SSL fingerprint of the remote branch (shown during startup) or an empty string.",
+								"type":        "string",
+							},
+						},
+					},
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns an empty body if successful.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	s["paths"].(map[string]interface{})["/v1/admin/{tree}/mapping"] = map[string]interface{}{
+		"post": map[string]interface{}{
+			"summary":     "Add a new mapping.",
+			"description": "Add a new mapping to the tree.",
+			"consumes": []string{
+				"application/json",
+			},
+			"produces": []string{
+				"text/plain",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "data",
+					"in":          "body",
+					"description": "Definition of the new branch.",
+					"required":    true,
+					"schema": map[string]interface{}{
+						"type": "object",
+						"properties": map[string]interface{}{
+							"branch": map[string]interface{}{
+								"description": "Name of the known remote branch.",
+								"type":        "string",
+							},
+							"dir": map[string]interface{}{
+								"description": "Tree directory which should hold the branch root.",
+								"type":        "string",
+							},
+							"writable": map[string]interface{}{
+								"description": "Flag if the branch should be mapped as writable.",
+								"type":        "string",
+							},
+						},
+					},
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns an empty body if successful.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	// Add generic error object to definition
+
+	s["definitions"].(map[string]interface{})["Error"] = map[string]interface{}{
+		"description": "A human readable error mesage.",
+		"type":        "string",
+	}
+}

+ 283 - 0
api/v1/admin_test.go

@@ -0,0 +1,283 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package v1
+
+import (
+	"fmt"
+	"testing"
+
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+	"devt.de/krotik/rufs/config"
+)
+
+func TestAdminQuery(t *testing.T) {
+	queryURL := "http://localhost" + TESTPORT + EndpointAdmin
+
+	defer func() {
+
+		// Make sure all trees are removed
+
+		api.ResetTrees()
+	}()
+
+	// In the beginning there should be no trees
+
+	st, _, res := sendTestRequest(queryURL, "GET", nil)
+	if st != "200 OK" || res != "{}" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Create a new tree
+
+	st, _, res = sendTestRequest(queryURL, "POST", []byte("\"Hans1\""))
+	if st != "200 OK" || res != "" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Check a new tree was created
+
+	st, _, res = sendTestRequest(queryURL+"?refresh=Hans1", "GET", nil)
+	if st != "200 OK" || res != `
+{
+  "Hans1": {
+    "branches": [],
+    "tree": []
+  }
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Add a new branch
+
+	fooRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+	fooFP := footest.SSLFingerprint()
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/branch", "POST", []byte(fmt.Sprintf(`
+{
+	"branch" : "footest",
+	"rpc"    : %#v,
+	"fingerprint" : %#v
+}`, fooRPC, fooFP)))
+	if st != "200 OK" || res != "" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Check the branch was added
+
+	st, _, res = sendTestRequest(queryURL, "GET", nil)
+	if st != "200 OK" || res != fmt.Sprintf(`
+{
+  "Hans1": {
+    "branches": [
+      {
+        "branch": "footest",
+        "fingerprint": %#v,
+        "rpc": %#v
+      }
+    ],
+    "tree": []
+  }
+}`[1:], fooFP, fooRPC) {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Add a new mapping
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/mapping", "POST", []byte(`
+{
+	"dir" : "/",
+	"branch" : "footest",
+	"writeable" : false
+}`))
+	if st != "200 OK" || res != "" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Check the mapping was added
+
+	st, _, res = sendTestRequest(queryURL, "GET", nil)
+	if st != "200 OK" || res != fmt.Sprintf(`
+{
+  "Hans1": {
+    "branches": [
+      {
+        "branch": "footest",
+        "fingerprint": %#v,
+        "rpc": %#v
+      }
+    ],
+    "tree": [
+      {
+        "branch": "footest",
+        "path": "/",
+        "writeable": false
+      }
+    ]
+  }
+}`[1:], fooFP, fooRPC) {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Test error cases
+
+	st, _, res = sendTestRequest(queryURL, "POST", []byte(`""`))
+	if st != "400 Bad Request" || res != "Body must contain the tree name as a non-empty JSON string" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL, "POST", []byte(`x`))
+	if st != "400 Bad Request" || res != "Could not decode request body: invalid character 'x' looking for beginning of value" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	origConfig := api.TreeConfigTemplate
+	api.TreeConfigTemplate = nil
+
+	st, _, res = sendTestRequest(queryURL, "POST", []byte(`"xx"`))
+	if st != "400 Bad Request" || res != "Could not create new tree: Missing TreeSecret key in tree config" {
+		api.TreeConfigTemplate = origConfig
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	api.TreeConfigTemplate = origConfig
+
+	// Test error cases
+
+	origTrees := api.Trees
+
+	api.Trees = func() (map[string]*rufs.Tree, error) {
+		return nil, fmt.Errorf("Testerror")
+	}
+
+	st, _, res = sendTestRequest(queryURL, "GET", nil)
+	if st != "400 Bad Request" || res != "Testerror" {
+		api.Trees = origTrees
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	api.Trees = origTrees
+
+	st, _, res = sendTestRequest(queryURL, "POST", []byte("\"Hans1\""))
+	if st != "400 Bad Request" || res != "Could not add new tree: Tree Hans1 already exists" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans2/mapping", "POST", nil)
+	if st != "400 Bad Request" || res != "Unknown tree: Hans2" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/mapping", "POST", []byte("aaa"))
+	if st != "400 Bad Request" || res != "Could not decode request body: invalid character 'a' looking for beginning of value" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans2/", "POST", []byte("aaa"))
+	if st != "400 Bad Request" || res != "Need a tree name and a section (either branches or mapping)" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/branch", "POST", []byte(fmt.Sprintf(`
+{
+	"branch" : "footest",
+	"rpc"    : %#v,
+	"fingerprint" : %#v
+}`, fooRPC, fooFP)))
+	if st != "400 Bad Request" || res != "Could not add branch: Peer already registered: footest" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/branch", "POST", []byte(fmt.Sprintf(`
+{
+	"branch" : "footest",
+	"rpc2"    : %#v,
+	"fingerprint" : %#v
+}`, fooRPC, fooFP)))
+	if st != "400 Bad Request" || res != "Value for rpc is missing in posted data" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/mapping", "POST", []byte(`
+{
+	"dir" : "/",
+	"branch" : "footest2",
+	"writeable" : false
+}`))
+	if st != "400 Bad Request" || res != "Could not add branch: Unknown target node" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/mapping", "POST", []byte(`
+{
+	"dir" : "/",
+	"branch" : "footest2",
+	"writeable" : "test"
+}`))
+	if st != "400 Bad Request" || res != "Writeable value must be a boolean: strconv.ParseBool: parsing \"test\": invalid syntax" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Delete twice
+
+	if trees, err := api.Trees(); len(trees) != 1 || err != nil {
+		t.Error("Unexpected result:", trees, err)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1", "DELETE", nil)
+	if st != "200 OK" || res != "" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL, "DELETE", nil)
+	if st != "400 Bad Request" || res != "Need a tree name" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/meyer", "DELETE", nil)
+	if st != "400 Bad Request" || res != "Invalid resource specification: meyer" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	if trees, err := api.Trees(); len(trees) != 0 || err != nil {
+		t.Error("Unexpected result:", trees, err)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1", "DELETE", nil)
+	if st != "400 Bad Request" || res != "Could not remove tree: Tree Hans1 does not exist" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+}

+ 177 - 0
api/v1/dir.go

@@ -0,0 +1,177 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package v1
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"os"
+	"path"
+	"strconv"
+
+	"devt.de/krotik/common/stringutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+)
+
+/*
+EndpointDir is the dir endpoint URL (rooted). Handles everything
+under dir/...
+*/
+const EndpointDir = api.APIRoot + APIv1 + "/dir/"
+
+/*
+DirEndpointInst creates a new endpoint handler.
+*/
+func DirEndpointInst() api.RestEndpointHandler {
+	return &dirEndpoint{}
+}
+
+/*
+Handler object for dir operations.
+*/
+type dirEndpoint struct {
+	*api.DefaultEndpointHandler
+}
+
+/*
+HandleGET handles a dir query REST call.
+*/
+func (d *dirEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+	var tree *rufs.Tree
+	var ok, checksums bool
+	var err error
+	var dirs []string
+	var fis [][]os.FileInfo
+
+	if len(resources) == 0 {
+		http.Error(w, "Need at least a tree name",
+			http.StatusBadRequest)
+		return
+	}
+
+	if tree, ok, err = api.GetTree(resources[0]); err == nil && !ok {
+		err = fmt.Errorf("Unknown tree: %v", resources[0])
+	}
+
+	if err == nil {
+		var rex string
+
+		glob := r.URL.Query().Get("glob")
+		recursive, _ := strconv.ParseBool(r.URL.Query().Get("recursive"))
+		checksums, _ = strconv.ParseBool(r.URL.Query().Get("checksums"))
+
+		if rex, err = stringutil.GlobToRegex(glob); err == nil {
+
+			dirs, fis, err = tree.Dir(path.Join(resources[1:]...), rex, recursive, checksums)
+		}
+	}
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	data := make(map[string]interface{})
+
+	for i, d := range dirs {
+		var flist []map[string]interface{}
+
+		fi := fis[i]
+
+		for _, f := range fi {
+			toAdd := map[string]interface{}{
+				"name":  f.Name(),
+				"size":  f.Size(),
+				"isdir": f.IsDir(),
+			}
+
+			if checksums {
+				toAdd["checksum"] = f.(*rufs.FileInfo).Checksum()
+			}
+
+			flist = append(flist, toAdd)
+		}
+
+		data[d] = flist
+	}
+
+	// Write data
+
+	w.Header().Set("content-type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(data)
+}
+
+/*
+SwaggerDefs is used to describe the endpoint in swagger.
+*/
+func (d *dirEndpoint) SwaggerDefs(s map[string]interface{}) {
+
+	s["paths"].(map[string]interface{})["/v1/dir/{tree}/{path}"] = map[string]interface{}{
+		"get": map[string]interface{}{
+			"summary":     "Read a directory.",
+			"description": "List the contents of a directory.",
+			"produces": []string{
+				"text/plain",
+				"application/json",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "path",
+					"in":          "path",
+					"description": "Directory path.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "recursive",
+					"in":          "query",
+					"description": "Add listings of subdirectories.",
+					"required":    false,
+					"type":        "boolean",
+				},
+				{
+					"name":        "checksums",
+					"in":          "query",
+					"description": "Include file checksums.",
+					"required":    false,
+					"type":        "boolean",
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns a map of directories with a list of files as values.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	// Add generic error object to definition
+
+	s["definitions"].(map[string]interface{})["Error"] = map[string]interface{}{
+		"description": "A human readable error mesage.",
+		"type":        "string",
+	}
+}

+ 158 - 0
api/v1/dir_test.go

@@ -0,0 +1,158 @@
+package v1
+
+import (
+	"fmt"
+	"testing"
+
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+	"devt.de/krotik/rufs/config"
+)
+
+func TestDirQuery(t *testing.T) {
+	adminQueryURL := "http://localhost" + TESTPORT + EndpointAdmin
+
+	queryURL := "http://localhost" + TESTPORT + EndpointDir
+
+	// Setup a tree
+
+	defer func() {
+
+		// Make sure all trees are removed
+
+		api.ResetTrees()
+	}()
+
+	tree, err := rufs.NewTree(api.TreeConfigTemplate, api.TreeCertTemplate)
+	errorutil.AssertOk(err)
+
+	api.AddTree("Hans1", tree)
+
+	fooRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+	fooFP := footest.SSLFingerprint()
+
+	err = tree.AddBranch("footest", fooRPC, fooFP)
+	errorutil.AssertOk(err)
+
+	err = tree.AddMapping("/", "footest", false)
+	errorutil.AssertOk(err)
+
+	st, _, res := sendTestRequest(adminQueryURL, "GET", nil)
+	if st != "200 OK" || res != fmt.Sprintf(`
+{
+  "Hans1": {
+    "branches": [
+      {
+        "branch": "footest",
+        "fingerprint": %#v,
+        "rpc": %#v
+      }
+    ],
+    "tree": [
+      {
+        "branch": "footest",
+        "path": "/",
+        "writeable": false
+      }
+    ]
+  }
+}`[1:], fooFP, fooRPC) {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Get a directory listing
+
+	st, _, res = sendTestRequest(queryURL+"Hans1", "GET", nil)
+	if st != "200 OK" || res != `
+{
+  "/": [
+    {
+      "isdir": true,
+      "name": "sub1",
+      "size": 4096
+    },
+    {
+      "isdir": false,
+      "name": "test1",
+      "size": 10
+    },
+    {
+      "isdir": false,
+      "name": "test2",
+      "size": 10
+    }
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"Hans1/sub1", "GET", nil)
+	if st != "200 OK" || res != `
+{
+  "/sub1": [
+    {
+      "isdir": false,
+      "name": "test3",
+      "size": 17
+    }
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Test recursive with checksums
+
+	st, _, res = sendTestRequest(queryURL+"Hans1?recursive=TRUE&checksums=1", "GET", nil)
+	if st != "200 OK" || res != `
+{
+  "/": [
+    {
+      "checksum": "",
+      "isdir": true,
+      "name": "sub1",
+      "size": 4096
+    },
+    {
+      "checksum": "73b8af47",
+      "isdir": false,
+      "name": "test1",
+      "size": 10
+    },
+    {
+      "checksum": "b0c1fadd",
+      "isdir": false,
+      "name": "test2",
+      "size": 10
+    }
+  ],
+  "/sub1": [
+    {
+      "checksum": "f89782b1",
+      "isdir": false,
+      "name": "test3",
+      "size": 17
+    }
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	// Test error cases
+
+	st, _, res = sendTestRequest(queryURL, "GET", nil)
+	if st != "400 Bad Request" || res != "Need at least a tree name" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(queryURL+"dave", "GET", nil)
+	if st != "400 Bad Request" || res != "Unknown tree: dave" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+}

+ 839 - 0
api/v1/file.go

@@ -0,0 +1,839 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package v1
+
+import (
+	"encoding/json"
+	"fmt"
+	"mime/multipart"
+	"net/http"
+	"path"
+	"time"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/datautil"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/httputil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+)
+
+// Progress endpoint
+// =================
+
+/*
+Progress is a persisted data structure which contains the current
+progress of an ongoing operation.
+*/
+type Progress struct {
+	Op            string   // Operation which we show progress of
+	Subject       string   // Subject on which the operation is performed
+	Progress      int64    // Current progress of the ongoing operation (this is reset for each item)
+	TotalProgress int64    // Total progress required until current operation is finished
+	Item          int64    // Current processing item
+	TotalItems    int64    // Total number of items to process
+	Errors        []string // Any error messages
+}
+
+/*
+JSONString returns the progress object as a JSON string.
+*/
+func (p *Progress) JSONString() []byte {
+	ret, err := json.MarshalIndent(map[string]interface{}{
+		"operation":      p.Op,
+		"subject":        p.Subject,
+		"progress":       p.Progress,
+		"total_progress": p.TotalProgress,
+		"item":           p.Item,
+		"total_items":    p.TotalItems,
+		"errors":         p.Errors,
+	}, "", "    ")
+	errorutil.AssertOk(err)
+	return ret
+}
+
+/*
+ProgressMap contains information about copy progress.
+*/
+var ProgressMap = datautil.NewMapCache(100, 0)
+
+/*
+EndpointProgress is the progress endpoint URL (rooted). Handles everything
+under progress/...
+*/
+const EndpointProgress = api.APIRoot + APIv1 + "/progress/"
+
+/*
+ProgressEndpointInst creates a new endpoint handler.
+*/
+func ProgressEndpointInst() api.RestEndpointHandler {
+	return &progressEndpoint{}
+}
+
+/*
+Handler object for progress operations.
+*/
+type progressEndpoint struct {
+	*api.DefaultEndpointHandler
+}
+
+/*
+HandleGET handles a progress query REST call.
+*/
+func (f *progressEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+	var ok bool
+	var err error
+
+	if len(resources) < 2 {
+		http.Error(w, "Need a tree name and a progress ID",
+			http.StatusBadRequest)
+		return
+	}
+
+	if _, ok, err = api.GetTree(resources[0]); err == nil && !ok {
+		err = fmt.Errorf("Unknown tree: %v", resources[0])
+	}
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	p, ok := ProgressMap.Get(resources[0] + "#" + resources[1])
+
+	if !ok {
+		http.Error(w, fmt.Sprintf("Unknown progress ID: %v", resources[1]),
+			http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("content-type", "application/octet-stream")
+	w.Write(p.(*Progress).JSONString())
+}
+
+/*
+SwaggerDefs is used to describe the endpoint in swagger.
+*/
+func (f *progressEndpoint) SwaggerDefs(s map[string]interface{}) {
+
+	s["paths"].(map[string]interface{})["/v1/progress/{tree}/{progress_id}"] = map[string]interface{}{
+		"get": map[string]interface{}{
+			"summary":     "Request progress update.",
+			"description": "Return a progress object showing the progress of an ongoing operation.",
+			"produces": []string{
+				"text/plain",
+				"application/json",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "progress_id",
+					"in":          "path",
+					"description": "Id of progress object.",
+					"required":    true,
+					"type":        "string",
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns the requested progress object.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	// Add generic error object to definition
+
+	s["definitions"].(map[string]interface{})["Error"] = map[string]interface{}{
+		"description": "A human readable error mesage.",
+		"type":        "string",
+	}
+}
+
+// File endpoint
+// =============
+
+/*
+EndpointFile is the file endpoint URL (rooted). Handles everything
+under file/...
+*/
+const EndpointFile = api.APIRoot + APIv1 + "/file/"
+
+/*
+FileEndpointInst creates a new endpoint handler.
+*/
+func FileEndpointInst() api.RestEndpointHandler {
+	return &fileEndpoint{}
+}
+
+/*
+Handler object for file operations.
+*/
+type fileEndpoint struct {
+	*api.DefaultEndpointHandler
+}
+
+/*
+HandleGET handles a file query REST call.
+*/
+func (f *fileEndpoint) HandleGET(w http.ResponseWriter, r *http.Request, resources []string) {
+	var tree *rufs.Tree
+	var ok bool
+	var err error
+
+	if len(resources) < 2 {
+		http.Error(w, "Need a tree name and a file path",
+			http.StatusBadRequest)
+		return
+	}
+
+	if tree, ok, err = api.GetTree(resources[0]); err == nil && !ok {
+		err = fmt.Errorf("Unknown tree: %v", resources[0])
+	}
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("content-type", "application/octet-stream")
+
+	if err := tree.ReadFileToBuffer(path.Join(resources[1:]...), w); err != nil {
+		http.Error(w, fmt.Sprintf("Could not read file %v: %v", path.Join(resources[1:]...), err.Error()),
+			http.StatusBadRequest)
+		return
+	}
+}
+
+/*
+HandlePUT handles REST calls to modify / copy existing files.
+*/
+func (f *fileEndpoint) HandlePUT(w http.ResponseWriter, r *http.Request, resources []string) {
+	f.handleFileOp("PUT", w, r, resources)
+}
+
+/*
+HandleDELETE handles REST calls to delete existing files.
+*/
+func (f *fileEndpoint) HandleDELETE(w http.ResponseWriter, r *http.Request, resources []string) {
+	f.handleFileOp("DELETE", w, r, resources)
+}
+
+func (f *fileEndpoint) handleFileOp(requestType string, w http.ResponseWriter, r *http.Request, resources []string) {
+	var action string
+	var data, ret map[string]interface{}
+	var tree *rufs.Tree
+	var ok bool
+	var err error
+	var files []string
+
+	if len(resources) < 1 {
+		http.Error(w, "Need a tree name and a file path",
+			http.StatusBadRequest)
+		return
+	} else if len(resources) == 1 {
+		resources = append(resources, "/")
+	}
+
+	if tree, ok, err = api.GetTree(resources[0]); err == nil && !ok {
+		err = fmt.Errorf("Unknown tree: %v", resources[0])
+	}
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	ret = make(map[string]interface{})
+
+	if requestType == "DELETE" {
+
+		// See if the request contains a body with a list of files
+
+		err = json.NewDecoder(r.Body).Decode(&files)
+
+	} else {
+
+		// Unless it is a delete request we need an action command
+
+		err = json.NewDecoder(r.Body).Decode(&data)
+
+		if err != nil {
+			http.Error(w, fmt.Sprintf("Could not decode request body: %v", err.Error()),
+				http.StatusBadRequest)
+			return
+		}
+
+		actionObj, ok := data["action"]
+
+		if !ok {
+			http.Error(w, fmt.Sprintf("Action command is missing from request body"),
+				http.StatusBadRequest)
+			return
+		}
+
+		action = fmt.Sprint(actionObj)
+	}
+
+	fullPath := path.Join(resources[1:]...)
+	if fullPath != "/" {
+		fullPath = "/" + fullPath
+	}
+
+	dir, file := path.Split(fullPath)
+
+	if requestType == "DELETE" {
+
+		if len(files) == 0 {
+			_, err = tree.ItemOp(dir, map[string]string{
+				rufs.ItemOpAction: rufs.ItemOpActDelete,
+				rufs.ItemOpName:   file,
+			})
+
+		} else {
+
+			// Delete the files given in the body
+
+			for _, f := range files {
+				dir, file := path.Split(f)
+
+				if err == nil {
+					_, err = tree.ItemOp(dir, map[string]string{
+						rufs.ItemOpAction: rufs.ItemOpActDelete,
+						rufs.ItemOpName:   file,
+					})
+				}
+			}
+		}
+
+	} else if action == "rename" {
+
+		if newNamesParam, ok := data["newnames"]; ok {
+
+			if newNames, ok := newNamesParam.([]interface{}); !ok {
+				err = fmt.Errorf("Parameter newnames must be a list of filenames")
+			} else {
+
+				if filesParam, ok := data["files"]; !ok {
+					err = fmt.Errorf("Parameter files is missing from request body")
+				} else {
+
+					if filesList, ok := filesParam.([]interface{}); !ok {
+						err = fmt.Errorf("Parameter files must be a list of files")
+					} else {
+
+						for i, f := range filesList {
+							dir, file := path.Split(fmt.Sprint(f))
+
+							if err == nil {
+								_, err = tree.ItemOp(dir, map[string]string{
+									rufs.ItemOpAction:  rufs.ItemOpActRename,
+									rufs.ItemOpName:    file,
+									rufs.ItemOpNewName: fmt.Sprint(newNames[i]),
+								})
+							}
+						}
+					}
+				}
+			}
+
+		} else {
+
+			newName, ok := data["newname"]
+
+			if !ok {
+				err = fmt.Errorf("Parameter newname is missing from request body")
+
+			} else {
+
+				_, err = tree.ItemOp(dir, map[string]string{
+					rufs.ItemOpAction:  rufs.ItemOpActRename,
+					rufs.ItemOpName:    file,
+					rufs.ItemOpNewName: fmt.Sprint(newName),
+				})
+			}
+		}
+
+	} else if action == "mkdir" {
+
+		_, err = tree.ItemOp(dir, map[string]string{
+			rufs.ItemOpAction: rufs.ItemOpActMkDir,
+			rufs.ItemOpName:   file,
+		})
+
+	} else if action == "copy" {
+
+		dest, ok := data["destination"]
+
+		if !ok {
+			err = fmt.Errorf("Parameter destination is missing from request body")
+
+		} else {
+
+			// Create file list
+
+			filesParam, hasFilesParam := data["files"]
+
+			if hasFilesParam {
+
+				if lf, ok := filesParam.([]interface{}); !ok {
+					err = fmt.Errorf("Parameter files must be a list of files")
+
+				} else {
+					files = make([]string, len(lf))
+
+					for i, f := range lf {
+						files[i] = fmt.Sprint(f)
+					}
+				}
+
+			} else {
+
+				files = []string{fullPath}
+			}
+
+			if err == nil {
+
+				// Create progress object
+
+				uuid := fmt.Sprintf("%x", cryptutil.GenerateUUID())
+				ret["progress_id"] = uuid
+
+				mapLookup := resources[0] + "#" + uuid
+
+				ProgressMap.Put(mapLookup, &Progress{
+					Op:            "Copy",
+					Subject:       "",
+					Progress:      0,
+					TotalProgress: 0,
+					Item:          0,
+					TotalItems:    int64(len(files)),
+					Errors:        []string{},
+				})
+
+				go func() {
+
+					err = tree.Copy(files, fmt.Sprint(dest),
+						func(file string, writtenBytes, totalBytes, currentFile, totalFiles int64) {
+
+							if p, ok := ProgressMap.Get(mapLookup); ok && writtenBytes > 0 {
+								p.(*Progress).Subject = file
+								p.(*Progress).Progress = writtenBytes
+								p.(*Progress).TotalProgress = totalBytes
+								p.(*Progress).Item = currentFile
+								p.(*Progress).TotalItems = totalFiles
+							}
+						})
+
+					if err != nil {
+						if p, ok := ProgressMap.Get(mapLookup); ok {
+							p.(*Progress).Errors = append(p.(*Progress).Errors, err.Error())
+						}
+					}
+				}()
+
+				// Wait a little bit so immediate errors are directly reported
+
+				time.Sleep(10 * time.Millisecond)
+			}
+		}
+
+	} else if action == "sync" {
+
+		dest, ok := data["destination"]
+
+		if !ok {
+			err = fmt.Errorf("Parameter destination is missing from request body")
+
+		} else {
+
+			uuid := fmt.Sprintf("%x", cryptutil.GenerateUUID())
+			ret["progress_id"] = uuid
+
+			mapLookup := resources[0] + "#" + uuid
+
+			ProgressMap.Put(mapLookup, &Progress{
+				Op:            "Sync",
+				Subject:       "",
+				Progress:      0,
+				TotalProgress: -1,
+				Item:          0,
+				TotalItems:    -1,
+				Errors:        []string{},
+			})
+
+			go func() {
+
+				err = tree.Sync(fullPath, fmt.Sprint(dest), true,
+					func(op, srcFile, dstFile string, writtenBytes, totalBytes, currentFile, totalFiles int64) {
+
+						if p, ok := ProgressMap.Get(mapLookup); ok && writtenBytes > 0 {
+
+							p.(*Progress).Op = op
+							p.(*Progress).Subject = srcFile
+							p.(*Progress).Progress = writtenBytes
+							p.(*Progress).TotalProgress = totalBytes
+							p.(*Progress).Item = currentFile
+							p.(*Progress).TotalItems = totalFiles
+						}
+					})
+
+				if err != nil {
+					if p, ok := ProgressMap.Get(mapLookup); ok {
+						p.(*Progress).Errors = append(p.(*Progress).Errors, err.Error())
+					}
+				}
+
+			}()
+
+			// Wait a little bit so immediate errors are directly reported
+
+			time.Sleep(10 * time.Millisecond)
+		}
+
+	} else {
+
+		err = fmt.Errorf("Unknown action: %v", action)
+	}
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Write data
+
+	w.Header().Set("content-type", "application/json; charset=utf-8")
+	json.NewEncoder(w).Encode(ret)
+}
+
+/*
+HandlePOST handles REST calls to create or overwrite a new file.
+*/
+func (f *fileEndpoint) HandlePOST(w http.ResponseWriter, r *http.Request, resources []string) {
+	var err error
+	var tree *rufs.Tree
+	var ok bool
+
+	if len(resources) < 1 {
+		http.Error(w, "Need a tree name and a file path",
+			http.StatusBadRequest)
+		return
+	}
+
+	if tree, ok, err = api.GetTree(resources[0]); err == nil && !ok {
+		err = fmt.Errorf("Unknown tree: %v", resources[0])
+	}
+
+	if err != nil {
+		http.Error(w, err.Error(), http.StatusBadRequest)
+		return
+	}
+
+	// Check we have the right request type
+
+	if r.MultipartForm == nil {
+		if err = r.ParseMultipartForm(32 << 20); err != nil {
+			http.Error(w, fmt.Sprintf("Could not read request body: %v", err.Error()),
+				http.StatusBadRequest)
+			return
+		}
+	}
+
+	if r.MultipartForm != nil && r.MultipartForm.File != nil {
+
+		// Check the files are in the form field uploadfile
+
+		files, ok := r.MultipartForm.File["uploadfile"]
+		if !ok {
+			http.Error(w, "Could not find 'uploadfile' form field",
+				http.StatusBadRequest)
+			return
+		}
+
+		for _, file := range files {
+			var f multipart.File
+
+			// Write out all send files
+
+			if f, err = file.Open(); err == nil {
+				err = tree.WriteFileFromBuffer(path.Join(path.Join(resources[1:]...), file.Filename), f)
+			}
+
+			if err != nil {
+				http.Error(w, fmt.Sprintf("Could not write file %v: %v", path.Join(resources[1:]...)+file.Filename, err.Error()),
+					http.StatusBadRequest)
+				return
+			}
+		}
+	}
+
+	if redirect := r.PostFormValue("redirect"); redirect != "" {
+
+		// Do the redirect - make sure it is a local redirect
+
+		if err = httputil.CheckLocalRedirect(redirect); err != nil {
+			http.Error(w, fmt.Sprintf("Could not redirect: %v", err.Error()),
+				http.StatusBadRequest)
+			return
+		}
+
+		http.Redirect(w, r, redirect, http.StatusFound)
+	}
+}
+
+/*
+SwaggerDefs is used to describe the endpoint in swagger.
+*/
+func (f *fileEndpoint) SwaggerDefs(s map[string]interface{}) {
+
+	s["paths"].(map[string]interface{})["/v1/file/{tree}/{path}"] = map[string]interface{}{
+		"get": map[string]interface{}{
+			"summary":     "Read a file.",
+			"description": "Return the contents of a file.",
+			"produces": []string{
+				"text/plain",
+				"application/octet-stream",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "path",
+					"in":          "path",
+					"description": "File path.",
+					"required":    true,
+					"type":        "string",
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns the content of the requested file.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+		"put": map[string]interface{}{
+			"summary":     "Perform a file operation.",
+			"description": "Perform a file operation like rename or copy.",
+			"consumes": []string{
+				"application/json",
+			},
+			"produces": []string{
+				"text/plain",
+				"application/json",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "path",
+					"in":          "path",
+					"description": "File path.",
+					"required":    true,
+					"type":        "string",
+				},
+
+				{
+					"name":        "operation",
+					"in":          "body",
+					"description": "Operation which should be executes",
+					"required":    true,
+					"schema": map[string]interface{}{
+						"type": "object",
+						"properties": map[string]interface{}{
+							"action": map[string]interface{}{
+								"description": "Action to perform.",
+								"type":        "string",
+								"enum": []string{
+									"rename",
+									"mkdir",
+									"copy",
+									"sync",
+								},
+							},
+							"newname": map[string]interface{}{
+								"description": "New filename when renaming a single file.",
+								"type":        "string",
+							},
+							"newnames": map[string]interface{}{
+								"description": "List of new file names when renaming multiple files using the files parameter.",
+								"type":        "array",
+								"items": map[string]interface{}{
+									"description": "New filename.",
+									"type":        "string",
+								},
+							},
+							"destination": map[string]interface{}{
+								"description": "Destination directory when copying files.",
+								"type":        "string",
+							},
+							"files": map[string]interface{}{
+								"description": "List of (full path) files which should be copied / renamed.",
+								"type":        "array",
+								"items": map[string]interface{}{
+									"description": "File (with full path) which should be copied / renamed.",
+									"type":        "string",
+								},
+							},
+						},
+					},
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns the content of the requested file.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+		"post": map[string]interface{}{
+			"summary":     "Upload a file.",
+			"description": "Upload or overwrite a file.",
+			"produces": []string{
+				"text/plain",
+			},
+			"consumes": []string{
+				"multipart/form-data",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "path",
+					"in":          "path",
+					"description": "File path.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "redirect",
+					"in":          "formData",
+					"description": "Page to redirect to after processing the request.",
+					"required":    false,
+					"type":        "string",
+				},
+				{
+					"name":        "uploadfile",
+					"in":          "formData",
+					"description": "File(s) to create / overwrite.",
+					"required":    true,
+					"type":        "file",
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Successful upload no redirect parameter given.",
+				},
+				"302": map[string]interface{}{
+					"description": "Successful upload - redirect according to the given redirect parameter.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+		"delete": map[string]interface{}{
+			"summary":     "Delete a file or directory.",
+			"description": "Delete a file or directory.",
+			"produces": []string{
+				"text/plain",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "path",
+					"in":          "path",
+					"description": "File or directory path.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "filelist",
+					"in":          "body",
+					"description": "List of (full path) files which should be deleted",
+					"required":    false,
+					"schema": map[string]interface{}{
+						"type": "array",
+						"items": map[string]interface{}{
+							"description": "File (with full path) which should be deleted.",
+							"type":        "string",
+						},
+					},
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns the content of the requested file.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+
+	// Add generic error object to definition
+
+	s["definitions"].(map[string]interface{})["Error"] = map[string]interface{}{
+		"description": "A human readable error mesage.",
+		"type":        "string",
+	}
+}

File diff suppressed because it is too large
+ 1087 - 0
api/v1/file_test.go


+ 66 - 0
api/v1/rest.go

@@ -0,0 +1,66 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package v1
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"devt.de/krotik/rufs/api"
+)
+
+/*
+APIv1 is the directory for version 1 of the API
+*/
+const APIv1 = "/v1"
+
+/*
+V1EndpointMap is a map of urls to endpoints for version 1 of the API
+*/
+var V1EndpointMap = map[string]api.RestEndpointInst{
+	EndpointAdmin:    AdminEndpointInst,
+	EndpointDir:      DirEndpointInst,
+	EndpointFile:     FileEndpointInst,
+	EndpointProgress: ProgressEndpointInst,
+	EndpointZip:      ZipEndpointInst,
+}
+
+// Helper functions
+// ================
+
+/*
+checkResources check given resources for a GET request.
+*/
+func checkResources(w http.ResponseWriter, resources []string, requiredMin int, requiredMax int, errorMsg string) bool {
+	if len(resources) < requiredMin {
+		http.Error(w, errorMsg, http.StatusBadRequest)
+		return false
+	} else if len(resources) > requiredMax {
+		http.Error(w, "Invalid resource specification: "+strings.Join(resources[1:], "/"), http.StatusBadRequest)
+		return false
+	}
+	return true
+}
+
+/*
+getMapValue extracts a value from a given map.
+*/
+func getMapValue(w http.ResponseWriter, data map[string]interface{}, key string) (string, bool) {
+
+	if val, ok := data[key]; ok && val != "" {
+		return fmt.Sprint(val), true
+	}
+
+	http.Error(w, fmt.Sprintf("Value for %v is missing in posted data", key), http.StatusBadRequest)
+
+	return "", false
+}

+ 316 - 0
api/v1/rest_test.go

@@ -0,0 +1,316 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package v1
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/common/httputil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+	"devt.de/krotik/rufs/config"
+)
+
+const TESTPORT = ":9040"
+
+// Main function for all tests in this package
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// Create a ssl certificate directory
+
+	if res, _ := fileutil.PathExists(certdir); res {
+		os.RemoveAll(certdir)
+	}
+
+	err := os.Mkdir(certdir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create client certificate
+
+	certFile := fmt.Sprintf("cert-client.pem")
+	keyFile := fmt.Sprintf("key-client.pem")
+	host := "localhost"
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(path.Join(certdir, certFile), path.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	// Set the default client certificate and configuration for the REST API
+
+	api.TreeConfigTemplate = map[string]interface{}{
+		config.TreeSecret: "123",
+	}
+
+	api.TreeCertTemplate = &cert
+
+	// Ensure logging is discarded
+
+	log.SetOutput(ioutil.Discard)
+
+	// Set up test branches
+
+	b1, err := createBranch("footest", "foo")
+	errorutil.AssertOk(err)
+
+	b2, err := createBranch("bartest", "bar")
+	errorutil.AssertOk(err)
+
+	footest = b1
+	bartest = b2
+
+	// Create some test files
+
+	ioutil.WriteFile("foo/test1", []byte("Test1 file"), 0770)
+	ioutil.WriteFile("foo/test2", []byte("Test2 file"), 0770)
+
+	os.Mkdir("foo/sub1", 0770)
+	ioutil.WriteFile("foo/sub1/test3", []byte("Sub dir test file"), 0770)
+
+	ioutil.WriteFile("bar/test1", []byte("Test3 file"), 0770)
+
+	// Start the server
+
+	hs, wg := startServer()
+	if hs == nil {
+		return
+	}
+
+	// Register endpoints for version 1
+	api.RegisterRestEndpoints(api.GeneralEndpointMap)
+	api.RegisterRestEndpoints(V1EndpointMap)
+
+	// Run the tests
+
+	res := m.Run()
+
+	// Teardown
+
+	stopServer(hs, wg)
+
+	// Shutdown the branches
+
+	errorutil.AssertOk(b1.Shutdown())
+	errorutil.AssertOk(b2.Shutdown())
+
+	// Remove all directories again
+
+	if err = os.RemoveAll(certdir); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("foo"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("bar"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+
+	os.Exit(res)
+}
+
+func TestSwaggerDefs(t *testing.T) {
+
+	// Test we can build swagger defs from the endpoint
+
+	data := map[string]interface{}{
+		"paths":       map[string]interface{}{},
+		"definitions": map[string]interface{}{},
+	}
+
+	for _, inst := range V1EndpointMap {
+		inst().SwaggerDefs(data)
+	}
+
+	// Show swagger output
+
+	/*
+		queryURL := "http://localhost" + TESTPORT + api.EndpointSwagger
+		_, _, res := sendTestRequest(queryURL, "GET", nil)
+		fmt.Println(res)
+	*/
+}
+
+/*
+Send a request to a HTTP test server
+*/
+func sendTestRequest(url string, method string, content []byte) (string, http.Header, string) {
+	var req *http.Request
+	var err error
+
+	ct := "application/json"
+
+	if method == "FORMPOST" {
+		method = "POST"
+		ct = "application/x-www-form-urlencoded"
+	}
+
+	if content != nil {
+		req, err = http.NewRequest(method, url, bytes.NewBuffer(content))
+	} else {
+		req, err = http.NewRequest(method, url, nil)
+	}
+	if err != nil {
+		panic(err)
+	}
+
+	req.Header.Set("Content-Type", ct)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+
+	body, _ := ioutil.ReadAll(resp.Body)
+	bodyStr := strings.Trim(string(body), " \n")
+
+	// Try json decoding first
+
+	out := bytes.Buffer{}
+	err = json.Indent(&out, []byte(bodyStr), "", "  ")
+	if err == nil {
+		return resp.Status, resp.Header, out.String()
+	}
+
+	// Just return the body
+
+	return resp.Status, resp.Header, bodyStr
+}
+
+/*
+Start a HTTP test server.
+*/
+func startServer() (*httputil.HTTPServer, *sync.WaitGroup) {
+	hs := &httputil.HTTPServer{}
+
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	go hs.RunHTTPServer(TESTPORT, &wg)
+
+	wg.Wait()
+
+	// Server is started
+
+	if hs.LastError != nil {
+		panic(hs.LastError)
+	}
+
+	return hs, &wg
+}
+
+/*
+Stop a started HTTP test server.
+*/
+func stopServer(hs *httputil.HTTPServer, wg *sync.WaitGroup) {
+
+	if hs.Running == true {
+
+		wg.Add(1)
+
+		// Server is shut down
+
+		hs.Shutdown()
+
+		wg.Wait()
+
+	} else {
+
+		panic("Server was not running as expected")
+	}
+}
+
+const certdir = "certs" // Directory for certificates
+var portCount = 0       // Port assignment counter for Branch ports
+
+var footest, bartest *rufs.Branch                       // Branches
+var branchConfigs = map[string]map[string]interface{}{} // All branch configs
+
+/*
+createBranch creates a new branch.
+*/
+func createBranch(name, dir string) (*rufs.Branch, error) {
+
+	// Create the path directory
+
+	if res, _ := fileutil.PathExists(dir); res {
+		os.RemoveAll(dir)
+	}
+
+	err := os.Mkdir(dir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create the certificate
+
+	portCount++
+	host := fmt.Sprintf("localhost:%v", 9020+portCount)
+
+	// Generate a certificate and private key
+
+	certFile := fmt.Sprintf("cert-%v.pem", portCount)
+	keyFile := fmt.Sprintf("key-%v.pem", portCount)
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(filepath.Join(certdir, certFile), filepath.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	// Create the Branch
+
+	cfg := map[string]interface{}{
+		config.BranchName:     name,
+		config.BranchSecret:   "123",
+		config.EnableReadOnly: false,
+		config.RPCHost:        "localhost",
+		config.RPCPort:        fmt.Sprint(9020 + portCount),
+		config.LocalFolder:    dir,
+	}
+
+	branchConfigs[name] = cfg
+
+	return rufs.NewBranch(cfg, &cert)
+}

+ 143 - 0
api/v1/zip.go

@@ -0,0 +1,143 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package v1
+
+import (
+	"archive/zip"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+)
+
+/*
+EndpointZip is the zip endpoint URL (rooted). Handles everything
+under zip/...
+*/
+const EndpointZip = api.APIRoot + APIv1 + "/zip/"
+
+/*
+ZipEndpointInst creates a new endpoint handler.
+*/
+func ZipEndpointInst() api.RestEndpointHandler {
+	return &zipEndpoint{}
+}
+
+/*
+Handler object for zip operations.
+*/
+type zipEndpoint struct {
+	*api.DefaultEndpointHandler
+}
+
+/*
+HandlePOST handles a zip query REST call.
+*/
+func (z *zipEndpoint) HandlePOST(w http.ResponseWriter, r *http.Request, resources []string) {
+	var tree *rufs.Tree
+	var data []string
+	var ok bool
+	var err error
+
+	if !checkResources(w, resources, 1, 1, "Need a tree name") {
+		return
+	}
+
+	if tree, ok, err = api.GetTree(resources[0]); err == nil && !ok {
+		http.Error(w, fmt.Sprintf("Unknown tree: %v", resources[0]), http.StatusBadRequest)
+		return
+	}
+
+	if err = r.ParseForm(); err == nil {
+		files := r.Form["files"]
+
+		if len(files) == 0 {
+			err = fmt.Errorf("Field 'files' should be a list of files as JSON encoded string")
+		} else {
+			err = json.NewDecoder(bytes.NewBufferString(files[0])).Decode(&data)
+		}
+	}
+
+	if err != nil {
+		http.Error(w, fmt.Sprintf("Could not decode request body: %v", err.Error()),
+			http.StatusBadRequest)
+		return
+	}
+
+	w.Header().Set("content-type", "application/octet-stream")
+	w.Header().Set("content-disposition", `attachment; filename="files.zip"`)
+
+	// Go through the list of files and stream the zip file
+
+	zipW := zip.NewWriter(w)
+
+	for _, f := range data {
+		writer, _ := zipW.Create(f)
+		tree.ReadFileToBuffer(f, writer)
+	}
+
+	zipW.Close()
+}
+
+/*
+SwaggerDefs is used to describe the endpoint in swagger.
+*/
+func (z *zipEndpoint) SwaggerDefs(s map[string]interface{}) {
+
+	s["paths"].(map[string]interface{})["/v1/zip/{tree}"] = map[string]interface{}{
+		"post": map[string]interface{}{
+			"summary":     "Create zip file from a list of files.",
+			"description": "Combine a list of given files into a single zip file.",
+			"produces": []string{
+				"text/plain",
+			},
+			"consumes": []string{
+				"application/x-www-form-urlencoded",
+			},
+			"parameters": []map[string]interface{}{
+				{
+					"name":        "tree",
+					"in":          "path",
+					"description": "Name of the tree.",
+					"required":    true,
+					"type":        "string",
+				},
+				{
+					"name":        "files",
+					"in":          "body",
+					"description": "JSON encoded list of (full path) files which should be zipped up",
+					"required":    true,
+					"schema": map[string]interface{}{
+						"type": "array",
+						"items": map[string]interface{}{
+							"description": "File (with full path) which should be included in the zip file.",
+							"type":        "string",
+						},
+					},
+				},
+			},
+			"responses": map[string]interface{}{
+				"200": map[string]interface{}{
+					"description": "Returns the content of the requested file.",
+				},
+				"default": map[string]interface{}{
+					"description": "Error response",
+					"schema": map[string]interface{}{
+						"$ref": "#/definitions/Error",
+					},
+				},
+			},
+		},
+	}
+}

+ 131 - 0
api/v1/zip_test.go

@@ -0,0 +1,131 @@
+package v1
+
+import (
+	"archive/zip"
+	"bytes"
+	"fmt"
+	"testing"
+
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+	"devt.de/krotik/rufs/config"
+)
+
+func TestZipDownload(t *testing.T) {
+	queryURL := "http://localhost" + TESTPORT + EndpointDir
+	zipURL := "http://localhost" + TESTPORT + EndpointZip
+
+	// Setup a tree
+
+	defer func() {
+
+		// Make sure all trees are removed
+
+		api.ResetTrees()
+	}()
+
+	tree, err := rufs.NewTree(api.TreeConfigTemplate, api.TreeCertTemplate)
+	errorutil.AssertOk(err)
+
+	api.AddTree("Hans1", tree)
+
+	fooRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+	fooFP := footest.SSLFingerprint()
+
+	err = tree.AddBranch("footest", fooRPC, fooFP)
+	errorutil.AssertOk(err)
+
+	err = tree.AddMapping("/", "footest", false)
+	errorutil.AssertOk(err)
+
+	st, _, res := sendTestRequest(queryURL+"Hans1?recursive=1", "GET", nil)
+	if st != "200 OK" || res != `
+{
+  "/": [
+    {
+      "isdir": true,
+      "name": "sub1",
+      "size": 4096
+    },
+    {
+      "isdir": false,
+      "name": "test1",
+      "size": 10
+    },
+    {
+      "isdir": false,
+      "name": "test2",
+      "size": 10
+    }
+  ],
+  "/sub1": [
+    {
+      "isdir": false,
+      "name": "test3",
+      "size": 17
+    }
+  ]
+}`[1:] {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(zipURL+"Hans1", "FORMPOST",
+		[]byte(`files=["/test1", "/test2", "/sub1/test3"]`))
+	if st != "200 OK" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	r, err := zip.NewReader(bytes.NewReader([]byte(res)), int64(len(res)))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if res := len(r.File); res != 3 {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	if r.File[0].Name != "/test1" {
+		t.Error("Unexpected result:", r.File[0].Name)
+		return
+	}
+
+	if r.File[1].Name != "/test2" {
+		t.Error("Unexpected result:", r.File[1].Name)
+		return
+	}
+
+	if r.File[2].Name != "/sub1/test3" {
+		t.Error("Unexpected result:", r.File[2].Name)
+		return
+	}
+
+	if r.File[2].UncompressedSize != 17 {
+		t.Error("Unexpected result:", r.File[2].UncompressedSize)
+		return
+	}
+
+	// Test error cases
+
+	st, _, res = sendTestRequest(zipURL, "POST", nil)
+	if st != "400 Bad Request" || res != "Need a tree name" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(zipURL+"dave", "POST", nil)
+	if st != "400 Bad Request" || res != "Unknown tree: dave" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+
+	st, _, res = sendTestRequest(zipURL+"Hans1", "POST", nil)
+	if st != "400 Bad Request" || res != "Could not decode request body: Field 'files' should be a list of files as JSON encoded string" {
+		t.Error("Unexpected response:", st, res)
+		return
+	}
+}

+ 7 - 0
attach_webzip.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Use this simple script to attach the web.zip to Rufs's executable.
+
+echo >> rufs
+echo "####WEBZIP####" >> rufs
+cat web.zip >> rufs

+ 766 - 0
branch.go

@@ -0,0 +1,766 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package rufs contains the main API to Rufs.
+
+Rufs is organized as a collection of branches. Each branch represents a physical
+file system structure which can be queried and updated by an authorized client.
+
+On the client side one or several branches are organized into a tree. The
+single branches can overlay each other. For example:
+
+Branch A
+/foo/A
+/foo/B
+/bar/C
+
+Branch B
+/foo/C
+/test/D
+
+Tree 1
+/myspace => Branch A, Branch B
+
+Accessing tree with:
+/myspace/foo/A gets file /foo/A from Branch A while
+/myspace/foo/C gets file /foo/C from Branch B
+
+Write operations go only to branches which are mapped as writing branches
+and who accept them (i.e. are not set to readonly on the side of the branch).
+*/
+package rufs
+
+import (
+	"bytes"
+	"crypto/tls"
+	"encoding/gob"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/common/pools"
+	"devt.de/krotik/common/stringutil"
+	"devt.de/krotik/rufs/config"
+	"devt.de/krotik/rufs/node"
+)
+
+func init() {
+
+	// Make sure we can use the relevant types in a gob operation
+
+	gob.Register([][]os.FileInfo{})
+	gob.Register(&FileInfo{})
+}
+
+/*
+Branch models a single exported branch in Rufs.
+*/
+type Branch struct {
+	rootPath string         // Local directory (absolute path) modeling the branch root
+	node     *node.RufsNode // Local RPC node
+	readonly bool           // Flag if this branch is readonly
+}
+
+/*
+NewBranch returns a new exported branch.
+*/
+func NewBranch(cfg map[string]interface{}, cert *tls.Certificate) (*Branch, error) {
+	var err error
+	var b *Branch
+
+	// Make sure the given config is ok
+
+	if err = config.CheckBranchExportConfig(cfg); err == nil {
+
+		// Create RPC server
+
+		addr := fmt.Sprintf("%v:%v", fileutil.ConfStr(cfg, config.RPCHost),
+			fileutil.ConfStr(cfg, config.RPCPort))
+
+		rn := node.NewNode(addr, fileutil.ConfStr(cfg, config.BranchName),
+			fileutil.ConfStr(cfg, config.BranchSecret), cert, nil)
+
+		// Start the rpc server
+
+		if err = rn.Start(cert); err == nil {
+			var rootPath string
+
+			//  Construct root path
+
+			if rootPath, err = filepath.Abs(fileutil.ConfStr(cfg, config.LocalFolder)); err == nil {
+				b = &Branch{rootPath, rn, fileutil.ConfBool(cfg, config.EnableReadOnly)}
+				rn.DataHandler = b.requestHandler
+			}
+		}
+	}
+
+	return b, err
+}
+
+/*
+Name returns the name of the branch.
+*/
+func (b *Branch) Name() string {
+	return b.node.Name()
+}
+
+/*
+SSLFingerprint returns the SSL fingerprint of the branch.
+*/
+func (b *Branch) SSLFingerprint() string {
+	return b.node.SSLFingerprint()
+}
+
+/*
+Shutdown shuts the branch down.
+*/
+func (b *Branch) Shutdown() error {
+	return b.node.Shutdown()
+}
+
+/*
+IsReadOnly returns if this branch is read-only.
+*/
+func (b *Branch) IsReadOnly() bool {
+	return b.readonly
+}
+
+/*
+checkReadOnly returns an error if this branch is read-only.
+*/
+func (b *Branch) checkReadOnly() error {
+	var err error
+
+	if b.IsReadOnly() {
+		err = fmt.Errorf("Branch %v is read-only", b.Name())
+	}
+
+	return err
+}
+
+// Branch API
+// ==========
+
+/*
+Dir returns file listings matching a given pattern of one or more directories.
+The contents of the given path is returned along with checksums if the checksum
+flag is specified. Optionally, also the contents of all subdirectories can be
+returned if the recursive flag is set. The return values is a list of traversed
+directories (platform-agnostic) and their corresponding contents.
+*/
+func (b *Branch) Dir(spath string, pattern string, recursive bool, checksums bool) ([]string, [][]os.FileInfo, error) {
+	var fis []os.FileInfo
+
+	// Compile pattern
+
+	re, err := regexp.Compile(pattern)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	createRufsFileInfos := func(dirname string, afis []os.FileInfo) []os.FileInfo {
+		var fis []os.FileInfo
+
+		fis = make([]os.FileInfo, 0, len(afis))
+
+		for _, fi := range afis {
+
+			// Append if it matches the pattern
+
+			if re.MatchString(fi.Name()) {
+				fis = append(fis, fi)
+			}
+		}
+
+		// Wrap normal file infos and calculate checksum if necessary
+
+		ret := WrapFileInfos(dirname, fis)
+
+		if checksums {
+			for _, fi := range fis {
+				if !fi.IsDir() {
+
+					// The sum is either there or not ... - access errors should
+					// be caught when trying to read the file
+
+					sum, _ := fileutil.CheckSumFileFast(filepath.Join(dirname, fi.Name()))
+
+					fi.(*FileInfo).FiChecksum = sum
+				}
+			}
+		}
+
+		return ret
+	}
+
+	subPath, err := b.constructSubPath(spath)
+
+	if err == nil {
+
+		if !recursive {
+
+			if fis, err = ioutil.ReadDir(subPath); err == nil {
+				return []string{spath},
+					[][]os.FileInfo{createRufsFileInfos(subPath, fis)}, nil
+			}
+
+		} else {
+
+			var rpaths []string
+			var rfis [][]os.FileInfo
+			var addSubDir func(string, string) error
+
+			// Recursive function to walk directories and symlinks
+			// in a platform-agnostic way
+
+			addSubDir = func(p string, rp string) error {
+				fis, err = ioutil.ReadDir(p)
+
+				if err == nil {
+					rpaths = append(rpaths, rp)
+
+					rfis = append(rfis, createRufsFileInfos(p, fis))
+
+					for _, fi := range fis {
+
+						if err == nil && fi.IsDir() {
+							err = addSubDir(filepath.Join(p, fi.Name()),
+								path.Join(rp, fi.Name()))
+						}
+					}
+				}
+
+				return err
+			}
+
+			if err = addSubDir(subPath, spath); err == nil {
+				return rpaths, rfis, nil
+			}
+		}
+	}
+
+	// Ignore any not exists errors
+
+	if os.IsNotExist(err) {
+		err = nil
+	}
+
+	return nil, nil, err
+}
+
+/*
+ReadFileToBuffer reads a complete file into a given buffer which implements
+io.Writer.
+*/
+func (b *Branch) ReadFileToBuffer(spath string, buf io.Writer) error {
+	var n int
+	var err error
+	var offset int64
+
+	readBuf := make([]byte, DefaultReadBufferSize)
+
+	for err == nil {
+		n, err = b.ReadFile(spath, readBuf, offset)
+
+		if err == nil {
+			_, err = buf.Write(readBuf[:n])
+
+			offset += int64(n)
+
+		} else if IsEOF(err) {
+
+			// We reached the end of the file
+
+			err = nil
+			break
+		}
+	}
+
+	return err
+}
+
+/*
+ReadFile reads up to len(p) bytes into p from the given offset. It
+returns the number of bytes read (0 <= n <= len(p)) and any error
+encountered.
+*/
+func (b *Branch) ReadFile(spath string, p []byte, offset int64) (int, error) {
+	var n int
+
+	subPath, err := b.constructSubPath(spath)
+
+	if err == nil {
+		var fi os.FileInfo
+
+		if fi, err = os.Stat(subPath); err == nil {
+
+			if fi.IsDir() {
+				err = fmt.Errorf("read /%v: is a directory", spath)
+
+			} else if err == nil {
+				var f *os.File
+
+				if f, err = os.Open(subPath); err == nil {
+					defer f.Close()
+
+					sr := io.NewSectionReader(f, 0, fi.Size())
+
+					if _, err = sr.Seek(offset, io.SeekStart); err == nil {
+						n, err = sr.Read(p)
+					}
+				}
+			}
+		}
+	}
+
+	return n, err
+}
+
+/*
+WriteFileFromBuffer writes a complete file from a given buffer which implements
+io.Reader.
+*/
+func (b *Branch) WriteFileFromBuffer(spath string, buf io.Reader) error {
+	var err error
+	var offset int64
+
+	if err = b.checkReadOnly(); err == nil {
+
+		writeBuf := make([]byte, DefaultReadBufferSize)
+
+		for err == nil {
+			var n int
+
+			if n, err = buf.Read(writeBuf); err == nil {
+
+				_, err = b.WriteFile(spath, writeBuf[:n], offset)
+				offset += int64(n)
+
+			} else if IsEOF(err) {
+
+				// We reached the end of the file
+
+				b.WriteFile(spath, []byte{}, offset)
+
+				err = nil
+				break
+			}
+		}
+	}
+
+	return err
+}
+
+/*
+WriteFile writes p into the given file from the given offset. It
+returns the number of written bytes and any error encountered.
+*/
+func (b *Branch) WriteFile(spath string, p []byte, offset int64) (int, error) {
+	var n int
+	var m int64
+
+	if err := b.checkReadOnly(); err != nil {
+		return 0, err
+	}
+
+	buf := byteSlicePool.Get().([]byte)
+	defer func() {
+		byteSlicePool.Put(buf)
+	}()
+
+	growFile := func(f *os.File, n int64) {
+		var err error
+
+		toWrite := n
+
+		for err == nil && toWrite > 0 {
+			if toWrite > int64(DefaultReadBufferSize) {
+				_, err = f.Write(buf[:DefaultReadBufferSize])
+				toWrite -= int64(DefaultReadBufferSize)
+			} else {
+				_, err = f.Write(buf[:toWrite])
+				toWrite = 0
+			}
+		}
+	}
+
+	subPath, err := b.constructSubPath(spath)
+
+	if err == nil {
+		var fi os.FileInfo
+		var f *os.File
+
+		if fi, err = os.Stat(subPath); os.IsNotExist(err) {
+
+			// Ensure path exists
+
+			dir, _ := filepath.Split(subPath)
+
+			if err = os.MkdirAll(dir, 0755); err == nil {
+
+				// Create the file newly
+
+				if f, err = os.OpenFile(subPath, os.O_RDWR|os.O_CREATE, 0644); err == nil {
+					defer f.Close()
+
+					if offset > 0 {
+						growFile(f, offset)
+					}
+
+					m, err = io.Copy(f, bytes.NewBuffer(p))
+					n += int(m)
+				}
+			}
+
+		} else if err == nil {
+
+			// File does exist
+
+			if f, err := os.OpenFile(subPath, os.O_RDWR, 0644); err == nil {
+				defer f.Close()
+
+				if fi.Size() < offset {
+					f.Seek(fi.Size(), io.SeekStart)
+					growFile(f, offset-fi.Size())
+				} else {
+					f.Seek(offset, io.SeekStart)
+				}
+
+				m, err = io.Copy(f, bytes.NewBuffer(p))
+				errorutil.AssertOk(err)
+
+				n += int(m)
+			}
+		}
+	}
+
+	return n, err
+}
+
+/*
+ItemOp parameter
+*/
+const (
+	ItemOpAction  = "itemop_action"  // ItemOp action
+	ItemOpName    = "itemop_name"    // Item name
+	ItemOpNewName = "itemop_newname" // New item name
+)
+
+/*
+ItemOp actions
+*/
+const (
+	ItemOpActRename = "rename" // Rename a file or directory
+	ItemOpActDelete = "delete" // Delete a file or directory
+	ItemOpActMkDir  = "mkdir"  // Create a directory
+)
+
+/*
+ItemOp executes a file or directory specific operation which can either
+succeed or fail (e.g. rename or delete). Actions and parameters should
+be given in the opdata map.
+*/
+func (b *Branch) ItemOp(spath string, opdata map[string]string) (bool, error) {
+	res := false
+
+	if err := b.checkReadOnly(); err != nil {
+		return false, err
+	}
+
+	subPath, err := b.constructSubPath(spath)
+
+	if err == nil {
+
+		action := opdata[ItemOpAction]
+
+		fileFromOpData := func(key string) (string, error) {
+
+			// Make sure we are only dealing with files
+
+			_, name := filepath.Split(opdata[key])
+
+			if name == "" {
+				return "", fmt.Errorf("This operation requires a specific file or directory")
+			}
+
+			// Build the relative paths
+
+			return filepath.Join(filepath.FromSlash(subPath), name), nil
+		}
+
+		if action == ItemOpActMkDir {
+			var name string
+
+			// Make directory action
+
+			if name, err = fileFromOpData(ItemOpName); err == nil {
+
+				err = os.MkdirAll(name, 0755)
+			}
+
+		} else if action == ItemOpActRename {
+			var name, newname string
+
+			// Rename action
+
+			if name, err = fileFromOpData(ItemOpName); err == nil {
+				if newname, err = fileFromOpData(ItemOpNewName); err == nil {
+
+					err = os.Rename(name, newname)
+				}
+			}
+
+		} else if action == ItemOpActDelete {
+			var name string
+
+			// Delete action
+
+			if name, err = fileFromOpData(ItemOpName); err == nil {
+
+				del := func(name string) error {
+					var err error
+					if ok, _ := fileutil.PathExists(name); ok {
+						err = os.RemoveAll(name)
+					} else {
+						err = os.ErrNotExist
+					}
+					return err
+				}
+
+				if strings.Contains(name, "*") {
+					var rex string
+
+					// We have a wildcard
+
+					rootdir, glob := filepath.Split(name)
+
+					// Create a regex from the given glob expression
+
+					if rex, err = stringutil.GlobToRegex(glob); err == nil {
+						var dirs []string
+						var fis [][]os.FileInfo
+
+						if dirs, fis, err = b.Dir(spath, rex, true, false); err == nil {
+
+							for i, dir := range dirs {
+
+								// Remove all files and dirs according to the wildcard
+
+								for _, fi := range fis[i] {
+									os.RemoveAll(filepath.Join(rootdir,
+										filepath.FromSlash(dir), fi.Name()))
+								}
+							}
+						}
+					}
+
+				} else {
+
+					err = del(name)
+				}
+			}
+		}
+
+		// Determine if we succeeded
+
+		res = err == nil || os.IsNotExist(err)
+	}
+
+	return res, err
+}
+
+// Request handling functions
+// ==========================
+
+/*
+DefaultReadBufferSize is the default size for file reading.
+*/
+var DefaultReadBufferSize = 1024 * 16
+
+/*
+bufferPool holds buffers which are used to marshal objects.
+*/
+var bufferPool = pools.NewByteBufferPool()
+
+/*
+byteSlicePool holds buffers which are used to read files
+*/
+var byteSlicePool = pools.NewByteSlicePool(DefaultReadBufferSize)
+
+/*
+Meta parameter
+*/
+const (
+	ParamAction    = "a" // Requested action
+	ParamPath      = "p" // Path string
+	ParamPattern   = "x" // Pattern string
+	ParamRecursive = "r" // Recursive flag
+	ParamChecksums = "c" // Checksum flag
+	ParamOffset    = "o" // Offset parameter
+	ParamSize      = "s" // Size parameter
+)
+
+/*
+Possible actions
+*/
+const (
+	OpDir    = "dir"    // Read the contents of a path
+	OpRead   = "read"   // Read the contents of a file
+	OpWrite  = "write"  // Read the contents of a file
+	OpItemOp = "itemop" // File or directory operation
+)
+
+/*
+requestHandler handles incoming requests from other branches or trees.
+*/
+func (b *Branch) requestHandler(ctrl map[string]string, data []byte) ([]byte, error) {
+	var err error
+	var res interface{}
+	var ret []byte
+
+	action := ctrl[ParamAction]
+
+	// Handle operation requests
+
+	if action == OpDir {
+		var dirs []string
+		var fis [][]os.FileInfo
+
+		dir := ctrl[ParamPath]
+		pattern := ctrl[ParamPattern]
+		rec := strings.ToLower(ctrl[ParamRecursive]) == "true"
+		sum := strings.ToLower(ctrl[ParamChecksums]) == "true"
+
+		if dirs, fis, err = b.Dir(dir, pattern, rec, sum); err == nil {
+			res = []interface{}{dirs, fis}
+		}
+
+	} else if action == OpItemOp {
+
+		res, err = b.ItemOp(ctrl[ParamPath], ctrl)
+
+	} else if action == OpRead {
+		var size, n int
+		var offset int64
+
+		spath := ctrl[ParamPath]
+
+		if size, err = strconv.Atoi(ctrl[ParamSize]); err == nil {
+			if offset, err = strconv.ParseInt(ctrl[ParamOffset], 10, 64); err == nil {
+
+				buf := byteSlicePool.Get().([]byte)
+				defer func() {
+					byteSlicePool.Put(buf)
+				}()
+
+				if len(buf) < size {
+
+					// Constantly requesting bigger buffers will
+					// eventually replace all default sized buffers
+
+					buf = make([]byte, size)
+				}
+
+				if n, err = b.ReadFile(spath, buf[:size], offset); err == nil {
+					res = []interface{}{n, buf[:size]}
+				}
+			}
+		}
+	} else if action == OpWrite {
+		var offset int64
+
+		spath := ctrl[ParamPath]
+		if offset, err = strconv.ParseInt(ctrl[ParamOffset], 10, 64); err == nil {
+
+			res, err = b.WriteFile(spath, data, offset)
+		}
+	}
+
+	// Send the response
+
+	if err == nil {
+
+		// Allocate a new encoding buffer - no need to lock as
+		// it is based on sync.Pool
+
+		// Pooled encoding buffers are used to keep expensive buffer
+		// reallocations to a minimum. It is better to allocate the
+		// actual response buffer once the response size is known.
+
+		bb := bufferPool.Get().(*bytes.Buffer)
+
+		if err = gob.NewEncoder(bb).Encode(res); err == nil {
+			toSend := bb.Bytes()
+
+			// Allocate the response array
+
+			ret = make([]byte, len(toSend))
+
+			// Copy encoded result into the response array
+
+			copy(ret, toSend)
+		}
+
+		// Return the encoding buffer back to the pool
+
+		go func() {
+			bb.Reset()
+			bufferPool.Put(bb)
+		}()
+	}
+
+	if err != nil {
+
+		// Ensure we don't leak local paths - this might not work in
+		// all situations and depends on the underlying os. In this
+		// error messages might include information on the full local
+		// path in error messages.
+
+		absRoot, _ := filepath.Abs(b.rootPath)
+		err = fmt.Errorf("%v", strings.Replace(err.Error(), absRoot, "", -1))
+	}
+
+	return ret, err
+}
+
+// Util functions
+// ==============
+
+func (b *Branch) constructSubPath(rpath string) (string, error) {
+
+	// Produce the actual subpath - this should also produce windows
+	// paths correctly (i.e. foo/bar -> C:\root\foo\bar)
+
+	subPath := filepath.Join(b.rootPath, filepath.FromSlash(rpath))
+
+	// Check that the new sub path is under the root path
+
+	absSubPath, err := filepath.Abs(subPath)
+
+	if err == nil {
+
+		if strings.HasPrefix(absSubPath, b.rootPath) {
+			return subPath, nil
+		}
+
+		err = fmt.Errorf("Requested path %v is outside of the branch", rpath)
+	}
+
+	return "", err
+}

+ 385 - 0
branch_test.go

@@ -0,0 +1,385 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package rufs
+
+import (
+	"bytes"
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/rufs/config"
+)
+
+const certdir = "certs" // Directory for certificates
+var portCount = 0       // Port assignment counter for Branch ports
+
+var footest, bartest *Branch                            // Branches
+var branchConfigs = map[string]map[string]interface{}{} // All branch configs
+var clientCert *tls.Certificate
+
+func TestReadOnlyBranch(t *testing.T) {
+
+	x, err := createBranch("footest3", "foo3", true)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	defer os.RemoveAll("foo3")
+
+	if err := x.WriteFileFromBuffer("", nil); err == nil || err.Error() != "Branch footest3 is read-only" {
+		t.Error("Unepxected result:", err)
+		return
+	}
+
+	if _, err := x.WriteFile("", nil, 0); err == nil || err.Error() != "Branch footest3 is read-only" {
+		t.Error("Unepxected result:", err)
+		return
+	}
+
+	if _, err := x.ItemOp("", nil); err == nil || err.Error() != "Branch footest3 is read-only" {
+		t.Error("Unepxected result:", err)
+		return
+	}
+
+	if res, _, err := x.Dir("/", "(", false, false); err == nil || err.Error() != "error parsing regexp: missing closing ): `(`" {
+		t.Error("Unepxected result:", res, err)
+		return
+	}
+}
+
+func TestTreeTraversal(t *testing.T) {
+
+	// Test create and shutdown
+
+	x, err := createBranch("footest2", "foo2", false)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if footest.SSLFingerprint() == "" {
+		t.Error("Branch should have a SSL fingerprint")
+		return
+	}
+
+	if res := footest.Name(); res != "footest" {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	x.Shutdown()
+
+	os.RemoveAll("foo2")
+
+	// Build up a tree from one branch
+
+	cfg := map[string]interface{}{
+		config.TreeSecret:     "123",
+		config.EnableReadOnly: false,
+	}
+
+	tree, _ := NewTree(cfg, clientCert)
+
+	branchRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+
+	if err := tree.AddBranch("footest", branchRPC, ""); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := tree.AddMapping("/1", "footest", false); err != nil {
+		t.Error(err)
+		return
+	}
+	if err := tree.AddMapping("/1/sub1", "footest", false); err != nil {
+		t.Error(err)
+		return
+	}
+	if err := tree.AddMapping("/2/3/4", "footest", false); err != nil {
+		t.Error(err)
+		return
+	}
+	if err := tree.AddMapping("/2/3", "footest", false); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// We should now have the following structure:
+	//
+	// /1/test1
+	// /1/test2
+	// /1/sub1/test3
+	// /1/sub1/test1
+	// /1/sub1/test2
+	// /1/sub1/sub1/test3
+	// /2/3/test1
+	// /2/3/test2
+	// /2/3/sub1/test3
+	// /2/3/4/test1
+	// /2/3/4/test2
+	// /2/3/4/sub1/test3
+
+	if res := fmt.Sprint(tree); res != `
+/: 
+  1/: footest(r)
+    sub1/: footest(r)
+  2/: 
+    3/: footest(r)
+      4/: footest(r)
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	// Test tree traversal (non recursive)
+
+	var res string
+
+	treeVisitor := func(item *treeItem, treePath string, branchPath []string, branches []string, writable []bool) {
+		// treePath is used for the result (to present to the user)
+		// branchPath is send to the branch
+		// branches are the branches on the current level
+		res += fmt.Sprintf("(%v) /%v: %v\n", treePath, strings.Join(branchPath, "/"), branches)
+	}
+
+	res = ""
+	tree.root.findPathBranches("/", createMappingPath("/"), false, treeVisitor)
+	if res != `(/) /: []
+` {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res = ""
+	tree.root.findPathBranches("/", createMappingPath("/1"), false, treeVisitor)
+	if res != `(/) /1: []
+(/1) /: [footest]
+` {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res = ""
+	tree.root.findPathBranches("/", createMappingPath("/1/sub1"), false, treeVisitor)
+	if res != `(/) /1/sub1: []
+(/1) /sub1: [footest]
+(/1/sub1) /: [footest]
+` {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res = ""
+	tree.root.findPathBranches("/", createMappingPath("/2/"), false, treeVisitor)
+	if res != `(/) /2: []
+(/2) /: []
+` {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res = ""
+	tree.root.findPathBranches("/", createMappingPath("/2/3/4/5/6"), false, treeVisitor)
+	if res != `(/) /2/3/4/5/6: []
+(/2) /3/4/5/6: []
+(/2/3) /4/5/6: [footest]
+(/2/3/4) /5/6: [footest]
+` {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	// Test tree traversal (recursive)
+
+	res = ""
+	tree.root.findPathBranches("/", createMappingPath("/1"), true, treeVisitor)
+	if res != `(/) /1: []
+(/1) /: [footest]
+(/1/sub1) /: [footest]
+` {
+		t.Error("Unexpected result:", res)
+		return
+	}
+}
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	unitTestModes = true // Unit tests will have standard file modes
+	defer func() {
+		unitTestModes = false
+	}()
+
+	// Create a ssl certificate directory
+
+	if res, _ := fileutil.PathExists(certdir); res {
+		os.RemoveAll(certdir)
+	}
+
+	err := os.Mkdir(certdir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create client certificate
+
+	certFile := fmt.Sprintf("cert-client.pem")
+	keyFile := fmt.Sprintf("key-client.pem")
+	host := "localhost"
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(path.Join(certdir, certFile), path.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	clientCert = &cert
+
+	// Ensure logging is discarded
+
+	log.SetOutput(ioutil.Discard)
+
+	// Set up test branches
+
+	b1, err := createBranch("footest", "foo", false)
+	errorutil.AssertOk(err)
+
+	b2, err := createBranch("bartest", "bar", false)
+	errorutil.AssertOk(err)
+
+	footest = b1
+	bartest = b2
+
+	// Create some test files
+
+	ioutil.WriteFile("foo/test1", []byte("Test1 file"), 0770)
+	ioutil.WriteFile("foo/test2", []byte("Test2 file"), 0770)
+
+	os.Mkdir("foo/sub1", 0770)
+	ioutil.WriteFile("foo/sub1/test3", []byte("Sub dir test file"), 0770)
+
+	ioutil.WriteFile("bar/test1", []byte("Test3 file"), 0770)
+
+	// Run the tests
+
+	res := m.Run()
+
+	// Shutdown the branches
+
+	errorutil.AssertOk(b1.Shutdown())
+	errorutil.AssertOk(b2.Shutdown())
+
+	// Remove all directories again
+
+	if err = os.RemoveAll(certdir); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("foo"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("bar"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+
+	os.Exit(res)
+}
+
+/*
+createBranch creates a new branch.
+*/
+func createBranch(name, dir string, readionly bool) (*Branch, error) {
+
+	// Create the path directory
+
+	if res, _ := fileutil.PathExists(dir); res {
+		os.RemoveAll(dir)
+	}
+
+	err := os.Mkdir(dir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create the certificate
+
+	portCount++
+	host := fmt.Sprintf("localhost:%v", 9020+portCount)
+
+	// Generate a certificate and private key
+
+	certFile := fmt.Sprintf("cert-%v.pem", portCount)
+	keyFile := fmt.Sprintf("key-%v.pem", portCount)
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(filepath.Join(certdir, certFile), filepath.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	// Create the Branch
+
+	config := map[string]interface{}{
+		config.BranchName:     name,
+		config.BranchSecret:   "123",
+		config.EnableReadOnly: readionly,
+		config.RPCHost:        "localhost",
+		config.RPCPort:        fmt.Sprint(9020 + portCount),
+		config.LocalFolder:    dir,
+	}
+
+	branchConfigs[name] = config
+
+	return NewBranch(config, &cert)
+}
+
+/*
+dirLocal reads a local directory and returns all found file names as a string.
+*/
+func dirLocal(dir string) string {
+	var buf bytes.Buffer
+
+	fis, err := ioutil.ReadDir(dir)
+	errorutil.AssertOk(err)
+
+	for _, fi := range fis {
+		buf.WriteString(fi.Name())
+		if fi.IsDir() {
+			buf.WriteString("(dir)")
+		} else {
+			buf.WriteString(fmt.Sprintf("(%v)", fi.Size()))
+		}
+		buf.WriteString(fmt.Sprintln())
+	}
+
+	return buf.String()
+}

+ 258 - 0
cli/client.go

@@ -0,0 +1,258 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"runtime"
+
+	"devt.de/krotik/common/datautil"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/common/termutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/config"
+	"devt.de/krotik/rufs/term"
+)
+
+/*
+DefaultMappingFile is the default mapping file for client trees
+*/
+const DefaultMappingFile = "rufs.mapping.json"
+
+/*
+clientCli handles the client command line.
+*/
+func clientCli() error {
+	var tree *rufs.Tree
+	var err error
+	var fuseMount, dokanMount, webExport *string
+
+	if runtime.GOOS == "linux" {
+		fuseMount = flag.String("fuse-mount", "", "Mount tree as FUSE filesystem at specified path (read-only)")
+	}
+	if runtime.GOOS == "windows" {
+		dokanMount = flag.String("dokan-mount", "", "Mount tree as DOKAN filesystem at specified path (read-only)")
+	}
+
+	webExport = flag.String("web", "", "Export the tree through a https interface on the specified host:port")
+
+	secretFile, certDir := commonCliOptions()
+
+	showHelp := flag.Bool("help", false, "Show this help message")
+
+	flag.Usage = func() {
+		fmt.Println()
+		fmt.Println(fmt.Sprintf("Usage of %s client [mapping file]", os.Args[0]))
+		fmt.Println()
+		flag.PrintDefaults()
+		fmt.Println()
+		fmt.Println("The mapping file assignes remote branches to the local tree.")
+		fmt.Println(fmt.Sprintf("The client tries to load %v if no mapping file is defined.",
+			DefaultMappingFile))
+		fmt.Println("It starts empty if no mapping file exists. The mapping file")
+		fmt.Println("should have the following json format:")
+		fmt.Println()
+		fmt.Println("{")
+		fmt.Println(`  "branches" : [`)
+		fmt.Println(`    {`)
+		fmt.Println(`      "branch"      : <branch name>,`)
+		fmt.Println(`      "rpc"         : <rpc interface>,`)
+		fmt.Println(`      "fingerprint" : <fingerprint>`)
+		fmt.Println(`    },`)
+		fmt.Println("    ...")
+		fmt.Println("  ],")
+		fmt.Println(`  "tree" : [`)
+		fmt.Println(`    {`)
+		fmt.Println(`      "path"      : <path>,`)
+		fmt.Println(`      "branch"    : <branch name>,`)
+		fmt.Println(`      "writeable" : <writable flag>`)
+		fmt.Println(`    },`)
+		fmt.Println("    ...")
+		fmt.Println("  ]")
+		fmt.Println("}")
+		fmt.Println()
+	}
+
+	flag.CommandLine.Parse(os.Args[2:])
+
+	if *showHelp {
+		flag.Usage()
+		return nil
+	}
+
+	// Load secret and ssl certificate
+
+	secret, cert, err := loadSecretAndCert(*secretFile, *certDir)
+	errorutil.AssertOk(err)
+
+	// Create config
+
+	cfg := datautil.MergeMaps(config.DefaultTreeConfig)
+	delete(cfg, config.TreeSecret)
+
+	cfg[config.TreeSecret] = secret
+
+	// Check for a mapping file
+
+	mappingFile := DefaultMappingFile
+
+	if len(flag.Args()) > 0 {
+		mappingFile = flag.Arg(0)
+	}
+
+	// Create the tree object
+
+	if tree, err = rufs.NewTree(cfg, cert); err == nil {
+
+		// Load mapping file
+
+		if ok, _ := fileutil.PathExists(mappingFile); ok {
+			var conf []byte
+
+			fmt.Println(fmt.Sprintf("Using mapping file: %s", mappingFile))
+
+			if conf, err = ioutil.ReadFile(mappingFile); err == nil {
+				tree.SetMapping(string(conf))
+			}
+
+		} else if webExport != nil && *webExport != "" {
+
+			err = fmt.Errorf("Need a mapping file when using web export")
+
+		} else if fuseMount != nil && *fuseMount != "" {
+
+			err = fmt.Errorf("Need a mapping file when using FUSE mount")
+
+		} else if dokanMount != nil && *dokanMount != "" {
+
+			err = fmt.Errorf("Need a mapping file when using DOKAN mount")
+		}
+
+		if err == nil {
+
+			// Check if we want a file system or a terminal
+
+			if webExport != nil && *webExport != "" {
+
+				err = setupWebExport(webExport, tree, certDir)
+
+			} else if fuseMount != nil && *fuseMount != "" {
+
+				err = setupFuseMount(fuseMount, tree)
+
+			} else if dokanMount != nil && *dokanMount != "" {
+
+				err = setupDokanMount(dokanMount, tree)
+
+			} else {
+
+				// Create the terminal
+
+				tt := term.NewTreeTerm(tree, os.Stdout)
+
+				// Add special store config command only available in the command line version
+
+				tt.AddCmd("storeconfig",
+					"storeconfig [local file]", "Store the current tree mapping in a local file",
+					func(tt *term.TreeTerm, arg ...string) (string, error) {
+						mf := DefaultMappingFile
+						if len(arg) > 0 {
+							mf = arg[0]
+						}
+						return "", ioutil.WriteFile(mf, []byte(tree.Config()), 0600)
+					})
+
+				// Run the terminal
+
+				clt, err := termutil.NewConsoleLineTerminal(os.Stdout)
+
+				if err == nil {
+					isExitLine := func(s string) bool {
+						return s == "exit" || s == "q" || s == "quit" || s == "bye" || s == "\x04"
+					}
+
+					// Add history functionality
+
+					clt, err = termutil.AddHistoryMixin(clt, ".rufs_client_history",
+						func(s string) bool {
+							return isExitLine(s)
+						})
+
+					if err == nil {
+						dictChooser := func(lineWords []string,
+							dictCache map[string]termutil.Dict) (termutil.Dict, error) {
+
+							// Simple dict chooser 1st level are available commands
+							// 2nd level are the contents of the current directory
+
+							if len(lineWords) <= 1 {
+								return dictCache["cmds"], nil
+							}
+
+							var suggestions []string
+
+							if _, fis, err := tree.Dir(tt.CurrentDir(), "", false,
+								false); err == nil {
+
+								for _, f := range fis[0] {
+									suggestions = append(suggestions, f.Name())
+								}
+							}
+
+							return termutil.NewWordListDict(suggestions), nil
+						}
+
+						dict := termutil.NewMultiWordDict(dictChooser, map[string]termutil.Dict{
+							"cmds": termutil.NewWordListDict(tt.Cmds()),
+						})
+
+						// Add auto complete
+
+						clt, err = termutil.AddAutoCompleteMixin(clt, dict)
+
+						if err == nil {
+							if err = clt.StartTerm(); err == nil {
+								var line string
+
+								defer clt.StopTerm()
+
+								fmt.Println("Type 'q' or 'quit' to exit the shell and '?' to get help")
+
+								line, err = clt.NextLine()
+								for err == nil && !isExitLine(line) {
+
+									// Process the entered line
+
+									res, terr := tt.Run(line)
+
+									if res != "" {
+										clt.WriteString(fmt.Sprintln(res))
+									}
+									if terr != nil {
+										clt.WriteString(fmt.Sprintln(terr.Error()))
+									}
+
+									line, err = clt.NextLine()
+								}
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return err
+}

+ 25 - 0
cli/dokan.go

@@ -0,0 +1,25 @@
+// +build !windows
+
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package main
+
+import "devt.de/krotik/rufs"
+
+/*
+setupDokanMount mounts Rufs as a DOKAN filesystem.
+*/
+func setupDokanMount(dokanMount *string, tree *rufs.Tree) error {
+
+	// Only works on Windows.
+
+	return nil
+}

+ 32 - 0
cli/dokan_windows.go

@@ -0,0 +1,32 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package main
+
+import (
+	"fmt"
+
+	"devt.de/krotik/rufs"
+)
+
+/*
+setupDokanMount mounts Rufs as a DOKAN filesystem.
+*/
+func setupDokanMount(dokanMount *string, tree *rufs.Tree) error {
+	var err error
+
+	// Create a FUSE mount
+
+	fmt.Println(fmt.Sprintf("Mounting: %s", *dokanMount))
+
+	// TODO ...
+
+	return err
+}

+ 25 - 0
cli/fuse.go

@@ -0,0 +1,25 @@
+// +build !linux
+
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package main
+
+import "devt.de/krotik/rufs"
+
+/*
+setupFuseMount mounts Rufs as a FUSE filesystem.
+*/
+func setupFuseMount(fuseMount *string, tree *rufs.Tree) error {
+
+	// Only works on Linux.
+
+	return nil
+}

+ 76 - 0
cli/fuse_linux.go

@@ -0,0 +1,76 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/export"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"github.com/hanwen/go-fuse/v2/fuse/nodefs"
+	"github.com/hanwen/go-fuse/v2/fuse/pathfs"
+)
+
+/*
+setupFuseMount mounts Rufs as a FUSE filesystem.
+*/
+func setupFuseMount(fuseMount *string, tree *rufs.Tree) error {
+	var err error
+	var server *fuse.Server
+
+	// Create a FUSE mount
+
+	fmt.Println(fmt.Sprintf("Mounting: %s", *fuseMount))
+
+	// Set up FUSE server
+
+	nfs := pathfs.NewPathNodeFs(&export.RufsFuse{
+		FileSystem: pathfs.NewDefaultFileSystem(),
+		Tree:       tree,
+	}, nil)
+
+	if server, _, err = nodefs.MountRoot(*fuseMount, nfs.Root(), nil); err != nil {
+		return err
+	}
+
+	// Add an unmount handler
+
+	// Attach SIGINT handler - on unix and windows this is send
+	// when the user presses ^C (Control-C).
+
+	sigchan := make(chan os.Signal)
+	signal.Notify(sigchan, syscall.SIGINT)
+
+	go func() {
+		for true {
+			signal := <-sigchan
+
+			if signal == syscall.SIGINT {
+				fmt.Println(fmt.Sprintf("Unmounting: %s", *fuseMount))
+				err := server.Unmount()
+				if err != nil {
+					fmt.Println(err)
+				}
+				break
+			}
+		}
+	}()
+
+	// Run FUSE server
+
+	server.Serve()
+
+	return err
+}

+ 207 - 0
cli/rufs.go

@@ -0,0 +1,207 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Rufs main entry point for the standalone server.
+*/
+package main
+
+import (
+	"crypto/tls"
+	"errors"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"time"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/rufs/config"
+)
+
+/*
+DefaultSecretFile is the default secret file which is used in server and client mode
+*/
+const DefaultSecretFile = "rufs.secret"
+
+/*
+DefaultSSLDir is the default directory containing the ssl key.pem and cert.pem files
+*/
+const DefaultSSLDir = "ssl"
+
+/*
+Main entry point for Rufs.
+*/
+func main() {
+	var err error
+
+	fmt.Println(fmt.Sprintf("Rufs %v", config.ProductVersion))
+
+	flag.Usage = func() {
+
+		// Print usage for tool selection
+
+		fmt.Println()
+		fmt.Println(fmt.Sprintf("Usage of %s [tool]", os.Args[0]))
+		fmt.Println()
+		fmt.Println("The tools are:")
+		fmt.Println()
+		fmt.Println("    server    Run as a server")
+		fmt.Println("    client    Run as a client")
+		fmt.Println()
+		fmt.Println(fmt.Sprintf("Use %s [tool] --help for more information about a tool.", os.Args[0]))
+		fmt.Println()
+	}
+
+	flag.Parse()
+
+	if len(flag.Args()) == 0 {
+		flag.Usage()
+		return
+	}
+
+	if flag.Args()[0] == "server" {
+
+		err = serverCli()
+
+	} else if flag.Args()[0] == "client" {
+
+		err = clientCli()
+
+	} else {
+		err = fmt.Errorf("Invalid tool")
+	}
+
+	if err != nil {
+		fmt.Println(fmt.Sprintf("Error: %v", err))
+	}
+}
+
+// Common code
+// ===========
+
+/*
+commonCliOptions returns common command line options which are relevant
+for both server and client.
+*/
+func commonCliOptions() (*string, *string) {
+	secretFile := flag.String("secret", DefaultSecretFile, "Secret file containing the secret token")
+	certDir := flag.String("ssl-dir", DefaultSSLDir, "Directory containing the ssl key.pem and cert.pem files")
+
+	return secretFile, certDir
+}
+
+/*
+loadSecretAndCert loads the secret string and the SSL key and certificate.
+*/
+func loadSecretAndCert(secretFile, certDir string) ([]byte, *tls.Certificate, error) {
+	var ok bool
+	var err error
+
+	// Load secret
+
+	if ok, _ = fileutil.PathExists(secretFile); !ok {
+		uuid := cryptutil.GenerateUUID()
+		err = ioutil.WriteFile(secretFile, uuid[:], 0600)
+	}
+
+	if err == nil {
+		var secret []byte
+
+		if secret, err = ioutil.ReadFile(secretFile); err == nil {
+
+			fmt.Println(fmt.Sprintf("Using secret from: %s", secretFile))
+
+			// Load ssl key and certificate
+
+			if ok, _ = fileutil.PathExists(certDir); !ok {
+
+				if err = os.MkdirAll(certDir, 0700); err == nil {
+
+					err = cryptutil.GenCert(certDir, "cert.pem", "key.pem", "localhost",
+						"", 365*24*time.Hour, false, 4096, "")
+				}
+			}
+
+			if err == nil {
+				var cert tls.Certificate
+
+				cert, err = tls.LoadX509KeyPair(filepath.Join(certDir, "cert.pem"),
+					filepath.Join(certDir, "key.pem"))
+
+				if err == nil {
+					fmt.Println(fmt.Sprintf("Using ssl key.pem and cert.pem from: %s", certDir))
+
+					return secret, &cert, nil
+				}
+			}
+		}
+	}
+
+	return nil, nil, err
+}
+
+/*
+externalIP returns the first found external IP
+*/
+func externalIP() (string, error) {
+	var ipstr string
+
+	ifaces, err := net.Interfaces()
+
+	if err == nil {
+
+	Loop:
+		for _, iface := range ifaces {
+			var addrs []net.Addr
+
+			if iface.Flags&net.FlagUp == 0 || iface.Flags&net.FlagLoopback != 0 {
+
+				// Ignore interfaces which are down or loopback devices
+
+				continue
+			}
+
+			if addrs, err = iface.Addrs(); err == nil {
+
+				// Go through all found addresses
+
+				for _, addr := range addrs {
+					var ip net.IP
+
+					switch v := addr.(type) {
+					case *net.IPNet:
+						ip = v.IP
+					case *net.IPAddr:
+						ip = v.IP
+					default:
+						continue
+					}
+
+					if !ip.IsLoopback() {
+						if ip = ip.To4(); ip != nil {
+							ipstr = ip.String()
+							break Loop
+						}
+					}
+				}
+			}
+		}
+	}
+
+	if ipstr == "" {
+		err = errors.New("No external interface found")
+	}
+
+	return ipstr, err
+}

+ 144 - 0
cli/server.go

@@ -0,0 +1,144 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"sync"
+	"syscall"
+
+	"devt.de/krotik/common/datautil"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/config"
+)
+
+/*
+DefaultServerConfigFile is the default config file when running in server mode
+*/
+const DefaultServerConfigFile = "rufs.server.json"
+
+/*
+serverCli handles the server command line.
+*/
+func serverCli() error {
+	var err error
+
+	serverConfigFile := flag.String("config", DefaultServerConfigFile, "Server configuration file")
+
+	secretFile, certDir := commonCliOptions()
+
+	showHelp := flag.Bool("help", false, "Show this help message")
+
+	flag.Usage = func() {
+		fmt.Println()
+		fmt.Println(fmt.Sprintf("Usage of %s server [options]", os.Args[0]))
+		fmt.Println()
+		flag.PrintDefaults()
+		fmt.Println()
+		fmt.Println("The server will automatically create a default config file and")
+		fmt.Println("default directories if nothing is specified.")
+		fmt.Println()
+	}
+
+	flag.CommandLine.Parse(os.Args[2:])
+
+	if *showHelp {
+		flag.Usage()
+		return nil
+	}
+
+	// Load secret and ssl certificate
+
+	secret, cert, err := loadSecretAndCert(*secretFile, *certDir)
+
+	if err == nil {
+
+		// Load configuration
+
+		var cfg map[string]interface{}
+
+		defaultConfig := datautil.MergeMaps(config.DefaultBranchExportConfig)
+		delete(defaultConfig, config.BranchSecret)
+
+		// Set environment specific values for default config
+
+		if ip, lerr := externalIP(); lerr == nil {
+			defaultConfig[config.BranchName] = ip
+			defaultConfig[config.RPCHost] = ip
+		}
+
+		cfg, err = fileutil.LoadConfig(*serverConfigFile, defaultConfig)
+		errorutil.AssertOk(err)
+
+		cfg[config.BranchSecret] = secret
+
+		fmt.Println(fmt.Sprintf("Using config: %s", *serverConfigFile))
+
+		// Ensure the local shared folder actually exists
+
+		if ok, _ := fileutil.PathExists(cfg[config.LocalFolder].(string)); !ok {
+			os.MkdirAll(cfg[config.LocalFolder].(string), 0777)
+		}
+
+		absLocalFolder, _ := filepath.Abs(cfg[config.LocalFolder].(string))
+		fmt.Println(fmt.Sprintf("Exporting folder: %s", absLocalFolder))
+
+		// We got everything together let's start
+
+		var branch *rufs.Branch
+
+		if branch, err = rufs.NewBranch(cfg, cert); err == nil {
+
+			// Attach SIGINT handler - on unix and windows this is send
+			// when the user presses ^C (Control-C).
+
+			sigchan := make(chan os.Signal)
+			signal.Notify(sigchan, syscall.SIGINT)
+
+			// Create a wait group to wait for the os signal
+
+			wg := sync.WaitGroup{}
+
+			// Kick off a polling thread which waits for the signal
+
+			go func() {
+				for true {
+					signal := <-sigchan
+
+					if signal == syscall.SIGINT {
+
+						// Shutdown the branch
+
+						branch.Shutdown()
+						break
+					}
+				}
+
+				// Done waiting main thread can exit
+
+				wg.Done()
+			}()
+
+			// Suspend main thread until branch is shutdown
+
+			wg.Add(1)
+			wg.Wait()
+		}
+	}
+
+	return err
+}

+ 219 - 0
cli/web.go

@@ -0,0 +1,219 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"unicode"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/common/httputil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+	"devt.de/krotik/rufs/api/v1"
+)
+
+/*
+webdir is the web directory which will contain all html files
+*/
+const webdir = "web"
+
+/*
+setupWebExport exports Rufs through a web interface
+*/
+func setupWebExport(webExport *string, tree *rufs.Tree, certDir *string) error {
+	var ok bool
+	var err error
+
+	// Register REST endpoints for version 1
+
+	api.RegisterRestEndpoints(v1.V1EndpointMap)
+	api.RegisterRestEndpoints(api.GeneralEndpointMap)
+
+	// Set the default tree
+
+	api.AddTree("default", tree)
+
+	// Ensure web folder
+
+	if ok, err = fileutil.PathExists(webdir); err == nil && !ok {
+		fmt.Println("Creating web folder")
+
+		err = extractWebFiles(webdir)
+	}
+
+	// Start the web server
+
+	if ok, _ = fileutil.PathExists(webdir); err == nil && ok {
+
+		fs := http.FileServer(http.Dir(webdir))
+
+		api.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+			fs.ServeHTTP(w, r)
+		})
+
+		// Start HTTPS server and enable REST API
+
+		hs := &httputil.HTTPServer{}
+
+		var wg sync.WaitGroup
+		wg.Add(1)
+
+		weblocString := *webExport
+		if strings.HasPrefix(weblocString, ":") {
+			weblocString = fmt.Sprintf("<any interface>%s", weblocString)
+		}
+
+		fmt.Println(fmt.Sprintf("Starting server on: https://%s", weblocString))
+
+		go hs.RunHTTPSServer(*certDir, "cert.pem", "key.pem",
+			*webExport, &wg)
+
+		// Wait until the server has started
+
+		wg.Wait()
+
+		if err = hs.LastError; err == nil {
+
+			// Read server certificate and write a fingerprint file
+
+			fpfile := filepath.Join(webdir, "fingerprint.json")
+
+			fmt.Println("Writing fingerprint file: ", fpfile)
+
+			certs, _ := cryptutil.ReadX509CertsFromFile(filepath.Join(*certDir, "cert.pem"))
+
+			if len(certs) > 0 {
+				buf := bytes.Buffer{}
+
+				buf.WriteString("{\n")
+				buf.WriteString(fmt.Sprintf(`  "md5"    : "%s",`, cryptutil.Md5CertFingerprint(certs[0])))
+				buf.WriteString("\n")
+				buf.WriteString(fmt.Sprintf(`  "sha1"   : "%s",`, cryptutil.Sha1CertFingerprint(certs[0])))
+				buf.WriteString("\n")
+				buf.WriteString(fmt.Sprintf(`  "sha256" : "%s"`, cryptutil.Sha256CertFingerprint(certs[0])))
+				buf.WriteString("\n")
+				buf.WriteString("}\n")
+
+				ioutil.WriteFile(fpfile, buf.Bytes(), 0644)
+			}
+
+			// Add to the wait group so we can wait for the shutdown
+
+			wg.Add(1)
+
+			fmt.Println("Waiting for shutdown")
+			wg.Wait()
+		}
+	}
+
+	return err
+}
+
+/*
+extractWebFiles extracts the web files from the executable.
+*/
+func extractWebFiles(webfolder string) error {
+	end := "####"
+	marker := fmt.Sprintf("%v%v%v", end, "WEBZIP", end)
+
+	exename, err := filepath.Abs(os.Args[0])
+	errorutil.AssertOk(err)
+
+	if ok, _ := fileutil.PathExists(exename); !ok {
+
+		// Try an optional .exe suffix which might work on Windows
+
+		exename += ".exe"
+	}
+
+	stat, err := os.Stat(exename)
+	if err != nil {
+		return err
+	}
+
+	// Open the executable
+
+	f, err := os.Open(exename)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	found := false
+	buf := make([]byte, 4096)
+	buf2 := make([]byte, len(marker)+10)
+
+	var pos int64
+
+	// Look for the marker which marks the beginning of the attached zip file
+
+	for i, err := f.Read(buf); err == nil; i, err = f.Read(buf) {
+
+		// Check if the marker could be in the read string
+
+		if strings.Contains(string(buf), "#") {
+
+			// Marker was found - read a bit more to ensure we got the full marker
+
+			if i2, err := f.Read(buf2); err == nil || err == io.EOF {
+				candidateString := string(append(buf, buf2...))
+
+				// Now determine the position if the zip file
+
+				if markerIndex := strings.Index(candidateString, marker); markerIndex >= 0 {
+					start := int64(markerIndex + len(marker))
+					for unicode.IsSpace(rune(candidateString[start])) || unicode.IsControl(rune(candidateString[start])) {
+						start++ // Skip final control characters \n or \r\n
+					}
+					pos += start
+					found = true
+					break
+				}
+
+				pos += int64(i2)
+			}
+		}
+
+		pos += int64(i)
+	}
+
+	if err == nil {
+		if found {
+
+			// Extract the zip
+
+			if _, err = f.Seek(pos, 0); err == nil {
+				zipLen := stat.Size() - pos
+
+				if err = os.MkdirAll(webfolder, 0755); err == nil {
+					err = fileutil.UnzipReader(io.NewSectionReader(f, pos, zipLen), zipLen, webfolder, false)
+				}
+			}
+
+		} else {
+
+			err = fmt.Errorf("Could not find web content marker in executable - invalid executable")
+		}
+	}
+
+	return err
+}

+ 83 - 0
config/config.go

@@ -0,0 +1,83 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package config
+
+import "fmt"
+
+/*
+ProductVersion is the current version of Rufs
+*/
+const ProductVersion = "0.0.0"
+
+/*
+Defaut configuration keys
+*/
+const (
+
+	// Branch configuration (export)
+
+	BranchName     = "BranchName"
+	BranchSecret   = "BranchSecret"
+	EnableReadOnly = "EnableReadOnly"
+	RPCHost        = "RPCHost"
+	RPCPort        = "RPCPort"
+	LocalFolder    = "LocalFolder"
+
+	// Tree configuration
+
+	TreeSecret = "TreeSecret"
+)
+
+/*
+DefaultBranchExportConfig is the default configuration for an exported branch
+*/
+var DefaultBranchExportConfig = map[string]interface{}{
+	BranchName:     "",      // Auto name (based on available network interface)
+	BranchSecret:   "",      // Secret needs to be provided by the client
+	EnableReadOnly: false,   // FS access is readonly for clients
+	RPCHost:        "",      // Auto (first available external interface)
+	RPCPort:        "9020",  // Communication port for this branch
+	LocalFolder:    "share", // Local folder which is being made available
+}
+
+/*
+DefaultTreeConfig is the default configuration for a tree which imports branches
+*/
+var DefaultTreeConfig = map[string]interface{}{
+	TreeSecret: "", // Secret needs to be provided by the client
+}
+
+// Helper functions
+// ================
+
+/*
+CheckBranchExportConfig checks a given branch export config.
+*/
+func CheckBranchExportConfig(config map[string]interface{}) error {
+	for k := range DefaultBranchExportConfig {
+		if _, ok := config[k]; !ok {
+			return fmt.Errorf("Missing %v key in branch export config", k)
+		}
+	}
+	return nil
+}
+
+/*
+CheckTreeConfig checks a given tree config.
+*/
+func CheckTreeConfig(config map[string]interface{}) error {
+	for k := range DefaultTreeConfig {
+		if _, ok := config[k]; !ok {
+			return fmt.Errorf("Missing %v key in tree config", k)
+		}
+	}
+	return nil
+}

+ 43 - 0
config/config_test.go

@@ -0,0 +1,43 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package config
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestConfig(t *testing.T) {
+
+	err := CheckBranchExportConfig(map[string]interface{}{})
+
+	if err == nil || strings.HasPrefix(err.Error(), "Unexpected result: Missing") {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	if err = CheckBranchExportConfig(DefaultBranchExportConfig); err != nil {
+		t.Error(err)
+		return
+	}
+
+	err = CheckTreeConfig(map[string]interface{}{})
+
+	if err == nil || strings.HasPrefix(err.Error(), "Unexpected result: Missing") {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	if err = CheckTreeConfig(DefaultTreeConfig); err != nil {
+		t.Error(err)
+		return
+	}
+}

+ 72 - 0
examples/tutorial/doc/tutorial.md

@@ -0,0 +1,72 @@
+Rufs Tutorial
+=============
+The following text will give you an overview of the main features of Rufs. It shows how to run simple local file share.
+
+The tutorial assumes you have downloaded Rufs and extracted it. Switch to the directory `examples/tutorial`. This example will demonstrate a simple local fileshare. Run `./start_server.sh` (Linux) or `start_server.bat` (Windows) to start the server. You should startup messages:
+```
+Rufs 1.0.0
+Using secret from: rufs.secret
+Using ssl key.pem and cert.pem from: ssl
+Using config: rufs.server.json
+Exporting folder: <absolute path to rufs>/examples/tutorial/run/share
+2019/08/10 21:12:59 local: Starting node local rpc server on: :9020
+2019/08/10 21:12:59 local: SSL fingerprint: <some long fingerprint>
+```
+The created `run` folder contains all runtime files. The created `rufs.secret` file is the `password` to the server and is never transmitted over the network. The client needs this file in order to communicate with the server.
+
+The next step is to start the client. The simplest client is the Rufs CLI. Run `./start_term_client.sh` (Linux) or `start_term_client.bat` (Windows) to start the command-line client. You should see startup messages and a prompt:
+```
+Rufs 1.0.0
+Using secret from: rufs.secret
+Using ssl key.pem and cert.pem from: ssl
+Using mapping file: rufs.mapping.json
+Type 'q' or 'quit' to exit the shell and '?' to get help
+>>>
+```
+The client will automatically connect to the running server. Run a few commands:
+```
+>>>branch
+local [94:2c:d4:70:e0:92:8c:41:77:c8:5a:1c:42:b0:71:4a:bb:de:3b:f5:10:f6:8f:80:e0:de:6c:97:44:0f:92:2b]
+```
+We can see from the `branch` command that only one branch is known in the moment. New branches can be added by writing `branch <branch name> <rpc port>` (e.g. `branch local localhost:9020`).
+```
+>>>mount
+/: local(w)
+```
+The `mount` command tells us that the branch is mounted as root and allows writing. Branches can be mounted by writing `mount <mount point> <branch name>` (e.g mount / local).
+
+Try mounting the local branch again in the subfolders `foo` and `foo2`:
+```
+>>>mount /foo local
+>>>mount /foo2 local
+```
+The `mount` command should now return the following:
+```
+>>>mount
+/: local(w)
+  foo/: local(w)
+  foo2/: local(w)  
+```
+Run the `dir` command to explore the new filesystem:
+```
+>>>dir
+/
+drwxrwxrwx  0 B   foo
+drwxrwxrwx  0 B   foo2
+-rw-r--r-- 12 B   test1.txt
+```
+Run `help` to see all available commands. You can quit the console by pressing `ctrl+d` or by running the `q` or `quit` command.
+
+Once you are finished exploring the console you can try the web interface by running `./start_web_client.sh` (Linux) or `start_web_client.bat` (Windows) in the `tutorial` folder. This should start a client running a webserver on `localhost:9090`
+
+The web client is a basic graphical interface and designed to be embedded in other web pages with customized styling. The basic interface looks like this:
+
+![](webclient_browser.png)
+
+The current branch mappings can be viewed under the `Mappings` tab:
+
+![](webclient_mappings.png)
+
+Files can be downloaded (by clicking on the filename) or uploaded (by dragging a file from the desktop). There are several options available via the menu:
+
+![](webclient_menu.png)

BIN
examples/tutorial/doc/webclient_browser.png


BIN
examples/tutorial/doc/webclient_mappings.png


BIN
examples/tutorial/doc/webclient_menu.png


+ 16 - 0
examples/tutorial/res/rufs.mapping.json

@@ -0,0 +1,16 @@
+{
+  "branches": [
+    {
+      "branch": "local",
+      "fingerprint": "",
+      "rpc": ":9020"
+    }
+  ],
+  "tree": [
+    {
+      "branch": "local",
+      "path": "/",
+      "writeable": true
+    }
+  ]
+}

+ 7 - 0
examples/tutorial/res/rufs.server.json

@@ -0,0 +1,7 @@
+{
+    "BranchName": "local",
+    "EnableReadOnly": false,
+    "LocalFolder": "share",
+    "RPCHost": "",
+    "RPCPort": "9020"
+}

+ 1 - 0
examples/tutorial/res/share/test1.txt

@@ -0,0 +1 @@
+test1 file

+ 9 - 0
examples/tutorial/start_server.bat

@@ -0,0 +1,9 @@
+@echo off
+
+if NOT EXIST run (
+  mkdir run
+  xcopy /e res\*.* run
+)
+
+cd run
+..\..\bin\rufs.exe server

+ 10 - 0
examples/tutorial/start_server.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+
+if ! [ -d "run" ]; then
+  mkdir -p run
+  cp -R res/* run
+fi
+
+cd run
+../../../rufs server

+ 2 - 0
examples/tutorial/start_term_client.bat

@@ -0,0 +1,2 @@
+@echo off
+..\..\bin\rufs.exe client

+ 5 - 0
examples/tutorial/start_term_client.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+
+cd run
+../../../rufs client

+ 2 - 0
examples/tutorial/start_web_client.bat

@@ -0,0 +1,2 @@
+@echo off
+..\..\bin\rufs.exe client -web :9000

+ 5 - 0
examples/tutorial/start_web_client.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+
+cd run
+../../../rufs client -web localhost:9090

+ 224 - 0
export/fuse.go

@@ -0,0 +1,224 @@
+// +build linux
+
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package export contains export bindings for Rufs.
+*/
+package export
+
+/*
+This file contains Rufs bindings for FUSE (Filesystem in Userspace) enabling
+a user to operate on Rufs as if it was a local file system.
+
+This uses GO-FUSE: https://github.com/hanwen/go-fuse
+Distributed under the New BSD License
+Copyright (c) 2010 the Go-FUSE Authors. All rights reserved.
+*/
+
+import (
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+
+	"devt.de/krotik/rufs"
+	"github.com/hanwen/go-fuse/v2/fuse"
+	"github.com/hanwen/go-fuse/v2/fuse/nodefs"
+	"github.com/hanwen/go-fuse/v2/fuse/pathfs"
+)
+
+/*
+RufsFuse is the Rufs specific FUSE filesystem API that uses paths rather
+than inodes.
+*/
+type RufsFuse struct {
+	pathfs.FileSystem
+	Tree *rufs.Tree
+}
+
+/*
+GetAttr is the main entry point, through which FUSE discovers which
+files and directories exist.
+*/
+func (rf *RufsFuse) GetAttr(name string, context *fuse.Context) (*fuse.Attr, fuse.Status) {
+
+	if name == "" {
+
+		// Mount point is always a directory
+
+		return &fuse.Attr{
+			Mode: fuse.S_IFDIR | 0755,
+		}, fuse.OK
+	}
+
+	var a *fuse.Attr
+
+	status := fuse.ENOENT
+
+	// Construct path and filename
+
+	name = path.Join("/", name)
+	dir, file := filepath.Split(name)
+
+	// Query the tree
+
+	_, fis, err := rf.Tree.Dir(dir, "", false, false)
+
+	if err != nil {
+		log.Print(err)
+		status = fuse.EIO
+	}
+
+	if len(fis) > 0 {
+
+		// Create attribute entries
+
+		for _, fi := range fis[0] {
+
+			if fi.Name() == file {
+
+				a = &fuse.Attr{
+					Mode: OSModeToFuseMode(fi.Mode()),
+					Size: uint64(fi.Size()),
+				}
+
+				status = fuse.OK
+			}
+		}
+	}
+
+	return a, status
+}
+
+/*
+OpenDir handles directories.
+*/
+func (rf *RufsFuse) OpenDir(name string,
+	context *fuse.Context) ([]fuse.DirEntry, fuse.Status) {
+	var c []fuse.DirEntry
+
+	// Construct path and filename
+
+	name = path.Join("/", name)
+	status := fuse.ENOENT
+
+	// Query the tree
+
+	_, fis, err := rf.Tree.Dir(name, "", false, false)
+
+	if err != nil {
+		LogError(err)
+		return nil, fuse.EIO
+	}
+
+	if len(fis) > 0 {
+
+		// Create entries
+
+		for _, fi := range fis[0] {
+			c = append(c, fuse.DirEntry{
+				Name: fi.Name(),
+				Mode: OSModeToFuseMode(fi.Mode()),
+			})
+		}
+
+		status = fuse.OK
+	}
+
+	return c, status
+}
+
+/*
+Open file handling.
+*/
+func (rf *RufsFuse) Open(name string, flags uint32, context *fuse.Context) (file nodefs.File, code fuse.Status) {
+	return &RufsFile{nodefs.NewDefaultFile(), path.Join("/", name), rf.Tree}, fuse.OK
+}
+
+// File related objects
+// ====================
+
+/*
+RufsFile models a file of Rufs.
+*/
+type RufsFile struct {
+	nodefs.File
+	name string
+	tree *rufs.Tree
+}
+
+/*
+Read reads a portion of the file.
+*/
+func (f *RufsFile) Read(buf []byte, off int64) (fuse.ReadResult, fuse.Status) {
+	var res fuse.ReadResult
+
+	status := fuse.OK
+
+	n, err := f.tree.ReadFile(f.name, buf, off)
+
+	if err != nil {
+		LogError(err)
+		status = fuse.EIO
+	} else {
+		res = &RufsReadResult{buf, n}
+	}
+
+	return res, status
+}
+
+/*
+RufsReadResult is an implementation of fuse.ReadResult.
+*/
+type RufsReadResult struct {
+	buf []byte
+	n   int
+}
+
+/*
+Bytes returns the raw bytes for the read.
+*/
+func (r *RufsReadResult) Bytes(buf []byte) ([]byte, fuse.Status) {
+	return r.buf, fuse.OK
+}
+
+/*
+Size returns how many bytes this return value takes at most.
+*/
+func (r *RufsReadResult) Size() int {
+	return r.n
+}
+
+/*
+Done is called after sending the data to the kernel.
+*/
+func (r *RufsReadResult) Done() {}
+
+// Helper functions
+// ================
+
+/*
+OSModeToFuseMode converts a given os.FileMode to a Fuse Mode
+*/
+func OSModeToFuseMode(fm os.FileMode) uint32 {
+	m := uint32(fm)
+
+	m = m & 0x0FFF // Remove special bits
+
+	if fm.IsDir() {
+		m = fuse.S_IFDIR | m
+	} else {
+		m = fuse.S_IFREG | m
+	}
+
+	return m
+}

+ 43 - 0
export/util.go

@@ -0,0 +1,43 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package export
+
+import "log"
+
+// Logging
+// =======
+
+/*
+Logger is a function which processes log messages
+*/
+type Logger func(v ...interface{})
+
+/*
+LogError is called if an error message is logged
+*/
+var LogError = Logger(log.Print)
+
+/*
+LogInfo is called if an info message is logged
+*/
+var LogInfo = Logger(log.Print)
+
+/*
+LogDebug is called if a debug message is logged
+(by default disabled)
+*/
+var LogDebug = Logger(LogNull)
+
+/*
+LogNull is a discarding logger to be used for disabling loggers
+*/
+var LogNull = func(v ...interface{}) {
+}

+ 158 - 0
fileinfo.go

@@ -0,0 +1,158 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package rufs
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+)
+
+/*
+Special unit test flag - use a common mode to gloss over OS specific
+defaults
+*/
+var unitTestModes = false
+
+/*
+FileInfo implements os.FileInfo in an platform-agnostic way
+*/
+type FileInfo struct {
+	FiName     string      // Base name
+	FiSize     int64       // Size in bytes
+	FiMode     os.FileMode // File mode bits
+	FiModTime  time.Time   // Modification time
+	FiChecksum string      // Checksum of files
+
+	// Private fields which will not be transferred via RPC
+
+	isSymLink     bool   // Flag if this is a symlink (unix)
+	symLinkTarget string // Target file/directory of the symlink
+}
+
+/*
+WrapFileInfo wraps a single os.FileInfo object in a serializable FileInfo.
+*/
+func WrapFileInfo(path string, i os.FileInfo) os.FileInfo {
+	var realPath string
+
+	// Check if we have a symlink
+
+	mode := i.Mode()
+	size := i.Size()
+
+	isSymlink := i.Mode()&os.ModeSymlink != 0
+
+	if isSymlink {
+		var err error
+
+		if realPath, err = filepath.EvalSymlinks(filepath.Join(path, i.Name())); err == nil {
+			var ri os.FileInfo
+			if ri, err = os.Stat(realPath); err == nil {
+
+				// Write in the size of the target and file mode
+
+				mode = ri.Mode()
+				size = ri.Size()
+			}
+		}
+	}
+
+	// Unit test fixed file modes
+
+	if unitTestModes {
+		mode = mode & os.ModeDir
+
+		if mode.IsDir() {
+			mode = mode | 0777
+			size = 4096
+		} else {
+			mode = mode | 0666
+		}
+	}
+
+	return &FileInfo{i.Name(), size, mode, i.ModTime(), "",
+		isSymlink, realPath}
+}
+
+/*
+WrapFileInfos wraps a list of os.FileInfo objects into a list of
+serializable FileInfo objects. This function will modify the given
+list.
+*/
+func WrapFileInfos(path string, is []os.FileInfo) []os.FileInfo {
+	for i, info := range is {
+		is[i] = WrapFileInfo(path, info)
+	}
+	return is
+}
+
+/*
+Name returns the base name.
+*/
+func (rfi *FileInfo) Name() string {
+	return rfi.FiName
+}
+
+/*
+Size returns the length in bytes.
+*/
+func (rfi *FileInfo) Size() int64 {
+	return rfi.FiSize
+}
+
+/*
+Mode returns the file mode bits.
+*/
+func (rfi *FileInfo) Mode() os.FileMode {
+	return rfi.FiMode
+}
+
+/*
+ModTime returns the modification time.
+*/
+func (rfi *FileInfo) ModTime() time.Time {
+	return rfi.FiModTime
+}
+
+/*
+Checksum returns the checksum of this file. May be an empty string if it was
+not calculated.
+*/
+func (rfi *FileInfo) Checksum() string {
+	return rfi.FiChecksum
+}
+
+/*
+IsDir returns if this is a directory.
+*/
+func (rfi *FileInfo) IsDir() bool {
+	return rfi.FiMode.IsDir()
+}
+
+/*
+Sys should return the underlying data source but will always return nil
+for FileInfo nodes.
+*/
+func (rfi *FileInfo) Sys() interface{} {
+	return nil
+}
+
+func (rfi *FileInfo) String() string {
+	sum := rfi.Checksum()
+	if sum != "" {
+		return fmt.Sprintf("%v %s [%v] %v (%v) - %v", rfi.Name(), sum, rfi.Size(),
+			rfi.Mode(), rfi.ModTime(), rfi.Sys())
+	}
+	return fmt.Sprintf("%v [%v] %v (%v) - %v", rfi.Name(), rfi.Size(),
+		rfi.Mode(), rfi.ModTime(), rfi.Sys())
+}

+ 53 - 0
fileinfo_test.go

@@ -0,0 +1,53 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package rufs
+
+import (
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+)
+
+func TestFileInfo(t *testing.T) {
+
+	oldUnitTestModes := unitTestModes
+	unitTestModes = false
+	defer func() {
+		unitTestModes = oldUnitTestModes
+	}()
+
+	fi := &FileInfo{"test", 500, os.FileMode(0764), time.Time{}, "123", false, ""}
+
+	if fi.String() != "test 123 [500] -rwxrw-r-- (0001-01-01 00:00:00 +0000 UTC) - <nil>" {
+		t.Error("Unexpected result:", fi)
+		return
+	}
+
+	fi = &FileInfo{"test", 500, os.FileMode(0764), time.Time{}, "", false, ""}
+
+	if fi.String() != "test [500] -rwxrw-r-- (0001-01-01 00:00:00 +0000 UTC) - <nil>" {
+		t.Error("Unexpected result:", fi)
+		return
+	}
+
+	ioutil.WriteFile("foo.txt", []byte("bar"), 0660)
+	defer os.Remove("foo.txt")
+
+	fi = &FileInfo{"foo.txt", 500, os.ModeSymlink, time.Time{}, "", false, ""}
+	fi = WrapFileInfo("./", fi).(*FileInfo)
+
+	if fi.String() != "foo.txt [3] -rw-rw---- (0001-01-01 00:00:00 +0000 UTC) - <nil>" &&
+		fi.String() != "foo.txt [3] -rw-rw-rw- (0001-01-01 00:00:00 +0000 UTC) - <nil>" {
+		t.Error("Unexpected result:", fi)
+		return
+	}
+}

+ 9 - 0
go.mod

@@ -0,0 +1,9 @@
+module devt.de/krotik/rufs
+
+go 1.12
+
+require (
+	devt.de/krotik/common v1.0.0
+	github.com/hanwen/go-fuse v1.0.0
+	github.com/hanwen/go-fuse/v2 v2.0.2
+)

+ 9 - 0
go.sum

@@ -0,0 +1,9 @@
+devt.de/krotik/common v1.0.0 h1:nMmFFkjqb8C/oFVfsEi39qnCUbu3J1FXg+FZn5gSOQU=
+devt.de/krotik/common v1.0.0/go.mod h1:X4nsS85DAxyHkwSg/Tc6+XC2zfmGeaVz+37F61+eSaI=
+github.com/hanwen/go-fuse v1.0.0 h1:GxS9Zrn6c35/BnfiVsZVWmsG803xwE7eVRDvcf/BEVc=
+github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
+github.com/hanwen/go-fuse/v2 v2.0.2 h1:BtsqKI5RXOqDMnTgpCb0IWgvRgGLJdqYVZ/Hm6KgKto=
+github.com/hanwen/go-fuse/v2 v2.0.2/go.mod h1:HH3ygZOoyRbP9y2q7y3+JM6hPL+Epe29IbWaS0UA81o=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522 h1:Ve1ORMCxvRmSXBwJK+t3Oy+V2vRW2OetUQBq4rJIkZE=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

+ 142 - 0
integration/rumble/dir.go

@@ -0,0 +1,142 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package rumble contains Rumble functions which interface with Rufs.
+*/
+package rumble
+
+import (
+	"fmt"
+	"os"
+	"regexp"
+
+	"devt.de/krotik/common/defs/rumble"
+	"devt.de/krotik/common/stringutil"
+	"devt.de/krotik/rufs/api"
+)
+
+// Function: dir
+// =============
+
+/*
+DirFunc queries a directory in a tree.
+*/
+type DirFunc struct {
+}
+
+/*
+Name returns the name of the function.
+*/
+func (f *DirFunc) Name() string {
+	return "fs.dir"
+}
+
+/*
+Validate is called for parameter validation and to reset the function state.
+*/
+func (f *DirFunc) Validate(argsNum int, rt rumble.Runtime) rumble.RuntimeError {
+	var err rumble.RuntimeError
+
+	if argsNum != 3 && argsNum != 4 {
+		err = rt.NewRuntimeError(rumble.ErrInvalidConstruct,
+			"Function dir requires 3 or 4 parameters: tree, a path, a glob expression and optionally a recursive flag")
+	}
+
+	return err
+}
+
+/*
+Execute executes the rumble function.
+*/
+func (f *DirFunc) Execute(argsVal []interface{}, vars rumble.Variables,
+	rt rumble.Runtime) (interface{}, rumble.RuntimeError) {
+
+	var res interface{}
+	var paths []string
+	var fiList [][]os.FileInfo
+
+	treeName := fmt.Sprint(argsVal[0])
+	path := fmt.Sprint(argsVal[1])
+	pattern := fmt.Sprint(argsVal[2])
+	recursive := argsVal[3] == true
+
+	conv := func(re *regexp.Regexp, fis []os.FileInfo) []interface{} {
+		r := make([]interface{}, 0, len(fis))
+
+		for _, fi := range fis {
+
+			if !fi.IsDir() && !re.MatchString(fi.Name()) {
+				continue
+			}
+
+			r = append(r, map[interface{}]interface{}{
+				"name":    fi.Name(),
+				"mode":    fmt.Sprint(fi.Mode()),
+				"modtime": fmt.Sprint(fi.ModTime()),
+				"isdir":   fi.IsDir(),
+				"size":    fi.Size(),
+			})
+		}
+
+		return r
+	}
+
+	tree, ok, err := api.GetTree(treeName)
+
+	if !ok {
+		if err == nil {
+			err = fmt.Errorf("Unknown tree: %v", treeName)
+		}
+	}
+
+	if err == nil {
+		var globPattern string
+
+		// Create regex for files
+
+		if globPattern, err = stringutil.GlobToRegex(pattern); err == nil {
+			var re *regexp.Regexp
+
+			if re, err = regexp.Compile(globPattern); err == nil {
+
+				// Query the file system
+
+				paths, fiList, err = tree.Dir(path, "", recursive, false)
+
+				pathData := make([]interface{}, 0, len(paths))
+				fisData := make([]interface{}, 0, len(paths))
+
+				// Convert the result into a Rumble data structure
+
+				for i := range paths {
+					fis := conv(re, fiList[i])
+
+					// If we have a regex then only include directories which have files
+
+					pathData = append(pathData, paths[i])
+					fisData = append(fisData, fis)
+				}
+
+				res = []interface{}{pathData, fisData}
+			}
+		}
+	}
+
+	if err != nil {
+
+		// Wrap error message in RuntimeError
+
+		err = rt.NewRuntimeError(rumble.ErrInvalidState,
+			fmt.Sprintf("Cannot list files: %v", err.Error()))
+	}
+
+	return res, err
+}

+ 272 - 0
integration/rumble/dir_test.go

@@ -0,0 +1,272 @@
+package rumble
+
+import (
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/defs/rumble"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/api"
+	"devt.de/krotik/rufs/config"
+)
+
+type mockRuntime struct {
+}
+
+func (mr *mockRuntime) NewRuntimeError(t error, d string) rumble.RuntimeError {
+	return fmt.Errorf("%v %v", t, d)
+}
+
+func TestDir(t *testing.T) {
+
+	// Setup a tree
+
+	defer func() {
+
+		// Make sure all trees are removed
+
+		api.ResetTrees()
+	}()
+
+	tree, err := rufs.NewTree(api.TreeConfigTemplate, api.TreeCertTemplate)
+	errorutil.AssertOk(err)
+
+	api.AddTree("Hans1", tree)
+
+	fooRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+	fooFP := footest.SSLFingerprint()
+
+	err = tree.AddBranch("footest", fooRPC, fooFP)
+	errorutil.AssertOk(err)
+
+	err = tree.AddMapping("/", "footest", false)
+	errorutil.AssertOk(err)
+
+	paths, infos, err := tree.Dir("/", "", true, true)
+	if res := rufs.DirResultToString(paths, infos); err != nil || res != `
+/
+drwxrwx--- 4.0 KiB sub1
+-rwxrwx---  10 B   test1 [73b8af47]
+-rwxrwx---  10 B   test2 [b0c1fadd]
+
+/sub1
+-rwxrwx--- 17 B   test3 [f89782b1]
+`[1:] {
+		t.Error("Unexpected result:", res, err)
+		return
+	}
+
+	df := &DirFunc{}
+	mr := &mockRuntime{}
+
+	if df.Name() != "fs.dir" {
+		t.Error("Unexpected result:", df.Name())
+		return
+	}
+
+	if err := df.Validate(1, mr); err == nil || err.Error() !=
+		"Invalid construct Function dir requires 3 or 4 parameters: tree, a path, a glob expression and optionally a recursive flag" {
+		t.Error(err)
+		return
+	}
+
+	if err := df.Validate(3, mr); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := df.Validate(4, mr); err != nil {
+		t.Error(err)
+		return
+	}
+
+	_, err = df.Execute([]interface{}{"Hans1", "/", "*.mp3", true}, nil, mr)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	res, err := df.Execute([]interface{}{"Hans1", "/", "", true}, nil, mr)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if fmt.Sprint(res.([]interface{})[0].([]interface{})[0]) != "/" {
+		t.Error("Unexpected result:", fmt.Sprint(res.([]interface{})[0].([]interface{})[0]))
+		return
+	}
+
+	if fmt.Sprint(res.([]interface{})[0].([]interface{})[1]) != "/sub1" {
+		t.Error("Unexpected result:", fmt.Sprint(res.([]interface{})[0].([]interface{})[1]))
+		return
+	}
+
+	// Test error messages
+
+	_, err = df.Execute([]interface{}{"Hans2", "/", "", true}, nil, mr)
+	if err == nil || err.Error() != "Invalid state Cannot list files: Unknown tree: Hans2" {
+		t.Error(err)
+		return
+	}
+}
+
+// Main function for all tests in this package
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// Create a ssl certificate directory
+
+	if res, _ := fileutil.PathExists(certdir); res {
+		os.RemoveAll(certdir)
+	}
+
+	err := os.Mkdir(certdir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create client certificate
+
+	certFile := fmt.Sprintf("cert-client.pem")
+	keyFile := fmt.Sprintf("key-client.pem")
+	host := "localhost"
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(path.Join(certdir, certFile), path.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	// Set the default client certificate and configuration for the REST API
+
+	api.TreeConfigTemplate = map[string]interface{}{
+		config.TreeSecret: "123",
+	}
+
+	api.TreeCertTemplate = &cert
+
+	// Ensure logging is discarded
+
+	log.SetOutput(ioutil.Discard)
+
+	// Set up test branches
+
+	b1, err := createBranch("footest", "foo")
+	errorutil.AssertOk(err)
+
+	b2, err := createBranch("bartest", "bar")
+	errorutil.AssertOk(err)
+
+	footest = b1
+	bartest = b2
+
+	// Create some test files
+
+	ioutil.WriteFile("foo/test1", []byte("Test1 file"), 0770)
+	ioutil.WriteFile("foo/test2", []byte("Test2 file"), 0770)
+
+	os.Mkdir("foo/sub1", 0770)
+	ioutil.WriteFile("foo/sub1/test3", []byte("Sub dir test file"), 0770)
+
+	ioutil.WriteFile("bar/test1", []byte("Test3 file"), 0770)
+
+	// Run the tests
+
+	res := m.Run()
+
+	// Shutdown the branches
+
+	errorutil.AssertOk(b1.Shutdown())
+	errorutil.AssertOk(b2.Shutdown())
+
+	// Remove all directories again
+
+	if err = os.RemoveAll(certdir); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("foo"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("bar"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+
+	os.Exit(res)
+}
+
+const certdir = "certs" // Directory for certificates
+var portCount = 0       // Port assignment counter for Branch ports
+
+var footest, bartest *rufs.Branch                       // Branches
+var branchConfigs = map[string]map[string]interface{}{} // All branch configs
+
+/*
+createBranch creates a new branch.
+*/
+func createBranch(name, dir string) (*rufs.Branch, error) {
+
+	// Create the path directory
+
+	if res, _ := fileutil.PathExists(dir); res {
+		os.RemoveAll(dir)
+	}
+
+	err := os.Mkdir(dir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create the certificate
+
+	portCount++
+	host := fmt.Sprintf("localhost:%v", 9020+portCount)
+
+	// Generate a certificate and private key
+
+	certFile := fmt.Sprintf("cert-%v.pem", portCount)
+	keyFile := fmt.Sprintf("key-%v.pem", portCount)
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(filepath.Join(certdir, certFile), filepath.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	// Create the Branch
+
+	config := map[string]interface{}{
+		config.BranchName:     name,
+		config.BranchSecret:   "123",
+		config.EnableReadOnly: false,
+		config.RPCHost:        "localhost",
+		config.RPCPort:        fmt.Sprint(9020 + portCount),
+		config.LocalFolder:    dir,
+	}
+
+	branchConfigs[name] = config
+
+	return rufs.NewBranch(config, &cert)
+}

+ 359 - 0
node/client.go

@@ -0,0 +1,359 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package node contains the network communication code for Rufs via RPC calls.
+*/
+package node
+
+import (
+	"crypto/tls"
+	"encoding/gob"
+	"fmt"
+	"io"
+	"net"
+	"net/rpc"
+	"os"
+	"sort"
+	"strings"
+	"sync"
+	"time"
+)
+
+func init() {
+
+	// Make sure we can use the relevant types in a gob operation
+
+	gob.Register(&RufsNodeToken{})
+	gob.Register(map[string]string{})
+}
+
+/*
+DialTimeout is the dial timeout for RPC connections
+*/
+var DialTimeout = 10 * time.Second
+
+/*
+RufsNodeToken is used to authenticate a node in the network to other nodes
+*/
+type RufsNodeToken struct {
+	NodeName string
+	NodeAuth string
+}
+
+/*
+Client is the client for the RPC API of a node.
+*/
+type Client struct {
+	token        *RufsNodeToken         // Token to be send to other nodes for authentication
+	rpc          string                 // This client's rpc network interface (may be empty in case of pure clients)
+	peers        map[string]string      // Map of node names to their rpc network interface
+	conns        map[string]*rpc.Client // Map of node names to network connections
+	fingerprints map[string]string      // Map of expected server certificate fingerprints
+	cert         *tls.Certificate       // Client certificate
+	maplock      *sync.RWMutex          // Lock for maps
+	redial       bool                   // Flag if this client is attempting a redial
+}
+
+/*
+SSLFingerprint returns the SSL fingerprint of the client.
+*/
+func (c *Client) SSLFingerprint() string {
+	var ret string
+	if c.cert != nil && c.cert.Certificate[0] != nil {
+		ret = fingerprint(c.cert.Certificate[0])
+	}
+	return ret
+}
+
+/*
+Shutdown closes all stored connections.
+*/
+func (c *Client) Shutdown() {
+	c.maplock.Lock()
+	defer c.maplock.Unlock()
+
+	for _, c := range c.conns {
+		c.Close()
+	}
+	c.conns = make(map[string]*rpc.Client)
+}
+
+/*
+RegisterPeer registers a new peer to communicate with. An empty fingerprint
+means that the client will accept any certificate from the server.
+*/
+func (c *Client) RegisterPeer(node string, rpc string, fingerprint string) error {
+
+	if _, ok := c.peers[node]; ok {
+		return fmt.Errorf("Peer already registered: %v", node)
+	} else if rpc == "" {
+		return fmt.Errorf("RPC interface must not be empty")
+	}
+
+	c.maplock.Lock()
+
+	c.peers[node] = rpc
+	delete(c.conns, node)
+	c.fingerprints[node] = fingerprint
+
+	c.maplock.Unlock()
+
+	return nil
+}
+
+/*
+Peers returns all registered peers and their expected fingerprints.
+*/
+func (c *Client) Peers() ([]string, []string) {
+	ret := make([]string, 0, len(c.peers))
+	fps := make([]string, len(c.peers))
+
+	c.maplock.Lock()
+	defer c.maplock.Unlock()
+
+	for k := range c.peers {
+		ret = append(ret, k)
+	}
+
+	sort.Strings(ret)
+
+	for i, node := range ret {
+		fps[i] = c.fingerprints[node]
+	}
+
+	return ret, fps
+}
+
+/*
+RemovePeer removes a registered peer.
+*/
+func (c *Client) RemovePeer(node string) {
+	c.maplock.Lock()
+	delete(c.peers, node)
+	delete(c.conns, node)
+	delete(c.fingerprints, node)
+	c.maplock.Unlock()
+}
+
+/*
+SendPing sends a ping to a node and returns the result. Second argument is
+optional if the target member is not a known peer. Should be an empty string
+in all other cases. Returns the answer, the fingerprint of the presented server
+certificate and any errors.
+*/
+func (c *Client) SendPing(node string, rpc string) ([]string, string, error) {
+	var ret []string
+	var fp string
+
+	if _, ok := c.peers[node]; !ok && rpc != "" {
+
+		// Add member temporary if it was not registered
+
+		c.maplock.Lock()
+		c.peers[node] = rpc
+		c.maplock.Unlock()
+
+		defer func() {
+			c.maplock.Lock()
+			delete(c.peers, node)
+			delete(c.conns, node)
+			delete(c.fingerprints, node)
+			c.maplock.Unlock()
+		}()
+	}
+
+	res, err := c.SendRequest(node, RPCPing, nil)
+
+	if res != nil && err == nil {
+		ret = res.([]string)
+		c.maplock.Lock()
+		fp = c.fingerprints[node]
+		c.maplock.Unlock()
+	}
+
+	return ret, fp, err
+}
+
+/*
+SendData sends a portion of data and some control information to a node and
+returns the result.
+*/
+func (c *Client) SendData(node string, ctrl map[string]string, data []byte) ([]byte, error) {
+
+	if _, ok := c.peers[node]; !ok {
+		return nil, fmt.Errorf("Unknown peer: %v", node)
+	}
+
+	res, err := c.SendRequest(node, RPCData, map[RequestArgument]interface{}{
+		RequestCTRL: ctrl,
+		RequestDATA: data,
+	})
+
+	if res != nil {
+		return res.([]byte), err
+	}
+
+	return nil, err
+}
+
+/*
+SendRequest sends a request to another node.
+*/
+func (c *Client) SendRequest(node string, remoteCall RPCFunction,
+	args map[RequestArgument]interface{}) (interface{}, error) {
+
+	var err error
+
+	// Function to categorize errors
+
+	handleError := func(err error) error {
+
+		if _, ok := err.(net.Error); ok {
+			return &Error{ErrNodeComm, err.Error(), false}
+		}
+
+		// Wrap remote errors in a proper error object
+
+		if err != nil && !strings.HasPrefix(err.Error(), "RufsError: ") {
+
+			// Check if the error is known to report that a file or directory
+			// does not exist.
+
+			err = &Error{ErrRemoteAction, err.Error(), err.Error() == os.ErrNotExist.Error()}
+		}
+
+		return err
+	}
+
+	c.maplock.Lock()
+	laddr, ok := c.peers[node]
+	c.maplock.Unlock()
+
+	if ok {
+
+		// Get network connection to the node
+
+		c.maplock.Lock()
+		conn, ok := c.conns[node]
+		c.maplock.Unlock()
+
+		if !ok {
+
+			// Create a new connection if necessary
+
+			nconn, err := net.DialTimeout("tcp", laddr, DialTimeout)
+
+			if err != nil {
+				LogDebug(c.token.NodeName, ": ",
+					fmt.Sprintf("- %v.%v (laddr=%v err=%v)",
+						node, remoteCall, laddr, err))
+				return nil, handleError(err)
+			}
+
+			if c.cert != nil && c.cert.Certificate[0] != nil {
+
+				// Wrap the conn in a TLS client
+
+				config := tls.Config{
+					Certificates:       []tls.Certificate{*c.cert},
+					InsecureSkipVerify: true,
+				}
+
+				tlsconn := tls.Client(nconn, &config)
+
+				// Do the handshake and look at the server certificate
+
+				tlsconn.Handshake()
+				rfp := fingerprint(tlsconn.ConnectionState().PeerCertificates[0].Raw)
+
+				c.maplock.Lock()
+				expected, _ := c.fingerprints[node]
+				c.maplock.Unlock()
+
+				if expected == "" {
+
+					// Accept the certificate and store it
+
+					c.maplock.Lock()
+					c.fingerprints[node] = rfp
+					c.maplock.Unlock()
+
+				} else if expected != rfp {
+
+					// Fingerprint was NOT verified
+
+					LogDebug(c.token.NodeName, ": ",
+						fmt.Sprintf("Not trusting %v (laddr=%v) presented fingerprint: %v expected fingerprint: %v", node, laddr, rfp, expected))
+
+					return nil, &Error{ErrUntrustedTarget, node, false}
+				}
+
+				LogDebug(c.token.NodeName, ": ",
+					fmt.Sprintf("%v (laddr=%v) has SSL fingerprint %v ", node, laddr, rfp))
+
+				nconn = tlsconn
+			}
+
+			conn = rpc.NewClient(nconn)
+
+			// Store the connection so it can be reused
+
+			c.maplock.Lock()
+			c.conns[node] = conn
+			c.maplock.Unlock()
+		}
+
+		// Assemble the request
+
+		request := map[RequestArgument]interface{}{
+			RequestTARGET: node,
+			RequestTOKEN:  c.token,
+		}
+
+		if args != nil {
+			for k, v := range args {
+				request[k] = v
+			}
+		}
+
+		var response interface{}
+
+		LogDebug(c.token.NodeName, ": ",
+			fmt.Sprintf("> %v.%v (laddr=%v)", node, remoteCall, laddr))
+
+		err = conn.Call("RufsServer."+string(remoteCall), request, &response)
+
+		if !c.redial && (err == rpc.ErrShutdown || err == io.EOF || err == io.ErrUnexpectedEOF) {
+
+			// Delete the closed connection and retry the request
+
+			c.maplock.Lock()
+			delete(c.conns, node)
+			c.redial = true // Set the redial flag to avoid a forever loop
+			c.maplock.Unlock()
+
+			return c.SendRequest(node, remoteCall, args)
+		}
+
+		// Reset redial flag
+
+		c.maplock.Lock()
+		c.redial = false
+		c.maplock.Unlock()
+
+		LogDebug(c.token.NodeName, ": ",
+			fmt.Sprintf("< %v.%v (err=%v)", node, remoteCall, err))
+
+		return response, handleError(err)
+	}
+
+	return nil, &Error{ErrUnknownTarget, node, false}
+}

+ 76 - 0
node/globals.go

@@ -0,0 +1,76 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package node
+
+import (
+	"errors"
+	"fmt"
+	"log"
+)
+
+// Logging
+// =======
+
+/*
+Logger is a function which processes log messages
+*/
+type Logger func(v ...interface{})
+
+/*
+LogInfo is called if an info message is logged
+*/
+var LogInfo = Logger(log.Print)
+
+/*
+LogDebug is called if a debug message is logged
+(by default disabled)
+*/
+var LogDebug = Logger(LogNull)
+
+/*
+LogNull is a discarding logger to be used for disabling loggers
+*/
+var LogNull = func(v ...interface{}) {
+}
+
+// Errors
+// ======
+
+/*
+Error is a network related error
+*/
+type Error struct {
+	Type       error  // Error type (to be used for equal checks)
+	Detail     string // Details of this error
+	IsNotExist bool   // Error is file or directory does not exist
+}
+
+/*
+Error returns a human-readable string representation of this error.
+*/
+func (ge *Error) Error() string {
+	if ge.Detail != "" {
+		return fmt.Sprintf("RufsError: %v (%v)", ge.Type, ge.Detail)
+	}
+
+	return fmt.Sprintf("RufsError: %v", ge.Type)
+}
+
+/*
+Network related error types
+*/
+var (
+	ErrNodeComm        = errors.New("Network error")
+	ErrRemoteAction    = errors.New("Remote error")
+	ErrUnknownTarget   = errors.New("Unknown target node")
+	ErrUntrustedTarget = errors.New("Unexpected SSL certificate from target node")
+	ErrInvalidToken    = errors.New("Invalid node token")
+)

+ 170 - 0
node/node.go

@@ -0,0 +1,170 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package node
+
+import (
+	"crypto/sha512"
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/rpc"
+	"sync"
+)
+
+/*
+RequestHandler is a function to handle incoming requests. A request has a
+control object which contains information on what the data is and how it
+should be used and the data itself. The request handler should return
+the result or an error.
+*/
+type RequestHandler func(ctrl map[string]string, data []byte) ([]byte, error)
+
+/*
+RufsNode is the management object for a node in the Rufs network.
+
+A RufsNode registers itself to the rpc server which is the global
+server object. Each node needs to have a unique name. Communication between nodes
+is secured by using a secret string which is never exchanged over the network
+and a hash generated token which identifies a member.
+
+Each RufsNode object contains a Client object which can be used to communicate
+with other nodes. This object should be used by pure clients - code which should
+communicate with the cluster without running an actual member.
+*/
+type RufsNode struct {
+	name        string           // Name of the node
+	secret      string           // Network wide secret
+	Client      *Client          // RPC client object
+	listener    net.Listener     // RPC server listener
+	wg          sync.WaitGroup   // RPC server Waitgroup for listener shutdown
+	DataHandler RequestHandler   // Handler function for data requests
+	cert        *tls.Certificate // Node certificate
+}
+
+/*
+NewNode create a new RufsNode object.
+*/
+func NewNode(rpcInterface string, name string, secret string, clientCert *tls.Certificate,
+	dataHandler RequestHandler) *RufsNode {
+
+	// Generate node token
+
+	token := &RufsNodeToken{name, fmt.Sprintf("%X", sha512.Sum512_224([]byte(name+secret)))}
+
+	rn := &RufsNode{name, secret, &Client{token, rpcInterface, make(map[string]string),
+		make(map[string]*rpc.Client), make(map[string]string), clientCert, &sync.RWMutex{}, false},
+		nil, sync.WaitGroup{}, dataHandler, clientCert}
+
+	return rn
+}
+
+/*
+NewClient create a new Client object.
+*/
+func NewClient(secret string, clientCert *tls.Certificate) *Client {
+	return NewNode("", "", secret, clientCert, nil).Client
+}
+
+// General node API
+// ================
+
+/*
+Name returns the name of the node.
+*/
+func (rn *RufsNode) Name() string {
+	return rn.name
+}
+
+/*
+SSLFingerprint returns the SSL fingerprint of the node.
+*/
+func (rn *RufsNode) SSLFingerprint() string {
+	var ret string
+	if rn.cert != nil && rn.cert.Certificate[0] != nil {
+		ret = fingerprint(rn.cert.Certificate[0])
+	}
+	return ret
+}
+
+/*
+LogInfo logs a node related message at info level.
+*/
+func (rn *RufsNode) LogInfo(v ...interface{}) {
+	LogInfo(rn.name, ": ", fmt.Sprint(v...))
+}
+
+/*
+Start starts process for this node.
+*/
+func (rn *RufsNode) Start(serverCert *tls.Certificate) error {
+
+	if _, ok := rufsServer.nodes[rn.name]; ok {
+		return fmt.Errorf("Cannot start node %s twice", rn.name)
+	}
+
+	rn.LogInfo("Starting node ", rn.name, " rpc server on: ", rn.Client.rpc)
+
+	l, err := net.Listen("tcp", rn.Client.rpc)
+	if err != nil {
+		return err
+	}
+
+	if serverCert != nil && serverCert.Certificate[0] != nil {
+		rn.cert = serverCert
+
+		rn.LogInfo("SSL fingerprint: ", rn.SSLFingerprint())
+
+		// Wrap the listener in a TLS listener
+
+		config := tls.Config{Certificates: []tls.Certificate{*serverCert}}
+
+		l = tls.NewListener(l, &config)
+	}
+
+	// Kick of the rpc listener
+
+	go func() {
+		rpc.Accept(l)
+		rn.wg.Done()
+		rn.LogInfo("Connection closed: ", rn.Client.rpc)
+	}()
+
+	rn.listener = l
+
+	// Register this node in the global server map
+
+	rufsServer.nodes[rn.name] = rn
+
+	return nil
+}
+
+/*
+Shutdown shuts the member manager rpc server for this cluster member down.
+*/
+func (rn *RufsNode) Shutdown() error {
+	var err error
+
+	// Close socket
+
+	if rn.listener != nil {
+		rn.LogInfo("Shutdown rpc server on: ", rn.Client.rpc)
+		rn.wg.Add(1)
+		err = rn.listener.Close()
+		rn.Client.Shutdown()
+		rn.listener = nil
+		rn.wg.Wait()
+		delete(rufsServer.nodes, rn.name)
+	} else {
+		LogDebug("Node ", rn.name, " already shut down")
+	}
+
+	return err
+}

+ 439 - 0
node/node_test.go

@@ -0,0 +1,439 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package node
+
+import (
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"testing"
+	"time"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/fileutil"
+)
+
+var consoleOutput = false
+var liveOutput = false
+
+type LogWriter struct {
+	w io.Writer
+}
+
+func (l LogWriter) Write(p []byte) (n int, err error) {
+	if liveOutput {
+		fmt.Print(string(p))
+	}
+	return l.w.Write(p)
+}
+
+const certdir = "certs"
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// Create output capture file
+
+	outFile, err := os.Create("out.txt")
+	if err != nil {
+		panic(err)
+	}
+
+	if res, _ := fileutil.PathExists(certdir); res {
+		os.RemoveAll(certdir)
+	}
+
+	err = os.Mkdir(certdir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Ensure logging is directed to the file
+
+	log.SetOutput(LogWriter{outFile})
+
+	// Run the tests
+
+	res := m.Run()
+
+	log.SetOutput(os.Stderr)
+
+	// Collected output
+
+	outFile.Sync()
+	outFile.Close()
+
+	stdout, err := ioutil.ReadFile("out.txt")
+	if err != nil {
+		panic(err)
+	}
+
+	// Handle collected output
+
+	if consoleOutput {
+		fmt.Println(string(stdout))
+	}
+
+	os.RemoveAll("out.txt")
+
+	err = os.RemoveAll(certdir)
+	if err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+
+	os.Exit(res)
+}
+
+/*
+createNodeNetwork creates a network of n nodes.
+*/
+func createNodeNetwork(n int) []*RufsNode {
+
+	var mms []*RufsNode
+
+	for i := 0; i < n; i++ {
+		host := fmt.Sprintf("localhost:%v", 9020+i)
+
+		// Generate a certificate and private key
+
+		certFile := fmt.Sprintf("cert-%v.pem", i)
+		keyFile := fmt.Sprintf("key-%v.pem", i)
+
+		err := cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+		if err != nil {
+			panic(err)
+		}
+
+		cert, err := tls.LoadX509KeyPair(path.Join(certdir, certFile), path.Join(certdir, keyFile))
+		if err != nil {
+			panic(err)
+		}
+
+		mms = append(mms, NewNode(host,
+			fmt.Sprintf("TestNode-%v", i), "test123", &cert, nil))
+	}
+
+	return mms
+}
+
+func TestReconnect(t *testing.T) {
+
+	// Debug logging
+
+	// liveOutput = true
+	// LogDebug = LogInfo
+	// defer func() { liveOutput = false }()
+
+	// Send a simple ping
+
+	nnet2 := createNodeNetwork(2)
+
+	nnet2[0].Start(nnet2[0].Client.cert) // Start the server with the client certificate
+	nnet2[1].Start(nnet2[1].Client.cert) // Start the server with the client certificate
+	defer nnet2[0].Shutdown()
+	defer nnet2[1].Shutdown()
+
+	var ctrlReceived map[string]string
+	var dataReceived string
+
+	nnet2[1].DataHandler = func(ctrl map[string]string, data []byte) ([]byte, error) {
+		dataReceived = string(data)
+		ctrlReceived = ctrl
+		return []byte("testack"), nil
+	}
+
+	// Register peer 1 on peer 0
+
+	nnet2[0].Client.RegisterPeer(nnet2[1].name, nnet2[1].Client.rpc, nnet2[1].SSLFingerprint())
+
+	// Send data successful
+
+	datares, err := nnet2[0].Client.SendData(nnet2[1].name, map[string]string{
+		"foo": "bar",
+	}, []byte("testmsg"))
+
+	if dataReceived != "testmsg" || fmt.Sprint(ctrlReceived) != "map[foo:bar]" ||
+		string(datares) != "testack" || err != nil {
+		t.Error("Unexpected result: ", ctrlReceived, dataReceived, datares, err)
+		return
+	}
+
+	ctrlReceived = nil
+	dataReceived = ""
+
+	// Shutdown peer 1
+
+	nnet2[1].Shutdown()
+
+	datares, err = nnet2[0].Client.SendData(nnet2[1].name, map[string]string{
+		"foo": "bar",
+	}, []byte("testmsg"))
+
+	if err == nil || err.Error() != "RufsError: Remote error (Unknown target node)" {
+		t.Error("Unexpected result: ", ctrlReceived, dataReceived, datares, err)
+		return
+	}
+
+	ctrlReceived = nil
+	dataReceived = ""
+
+	// Start again
+
+	nnet2[1].Start(nnet2[1].Client.cert)
+
+	datares, err = nnet2[0].Client.SendData(nnet2[1].name, map[string]string{
+		"foo": "bar",
+	}, []byte("testmsg"))
+
+	if dataReceived != "testmsg" || fmt.Sprint(ctrlReceived) != "map[foo:bar]" ||
+		string(datares) != "testack" || err != nil {
+		t.Error("Unexpected result: ", ctrlReceived, dataReceived, datares, err)
+		return
+	}
+}
+
+func Test2NodeNetwork(t *testing.T) {
+
+	// Debug logging
+
+	// liveOutput = true
+	// LogDebug = LogInfo
+	// defer func() { liveOutput = false }()
+
+	// Send a simple ping
+
+	nnet2 := createNodeNetwork(2)
+
+	nnet2[0].Start(nnet2[0].Client.cert) // Start the server with the client certificate
+	nnet2[1].Start(nnet2[1].Client.cert) // Start the server with the client certificate
+	defer nnet2[0].Shutdown()
+	defer nnet2[1].Shutdown()
+
+	// Check we are using the same certificates
+
+	if nnet2[0].Client.SSLFingerprint() != nnet2[0].SSLFingerprint() {
+		t.Errorf("Unexpected result:\n#%v\n#%v", nnet2[0].Client.SSLFingerprint(), nnet2[0].SSLFingerprint())
+		return
+	}
+
+	res, rfp, err := nnet2[0].Client.SendPing(nnet2[1].name, nnet2[1].Client.rpc)
+
+	if fmt.Sprint(res) != "[Pong]" || rfp != nnet2[1].SSLFingerprint() || err != nil {
+		t.Error("Unexpected result:", res, err)
+		return
+	}
+
+	if res := nnet2[1].Name(); res != "TestNode-1" {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	// Send a data request
+
+	var ctrlReceived map[string]string
+	var dataReceived string
+	var datares []byte
+
+	nnet2[1].DataHandler = func(ctrl map[string]string, data []byte) ([]byte, error) {
+		dataReceived = string(data)
+		ctrlReceived = ctrl
+		return []byte("testack"), nil
+	}
+
+	_, err = nnet2[0].Client.SendData(nnet2[1].name, map[string]string{
+		"foo": "bar",
+	}, []byte("testmsg"))
+
+	if err == nil || err.Error() != "Unknown peer: TestNode-1" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	// Register the peer
+
+	if err := nnet2[0].Client.RegisterPeer(nnet2[1].name, "", ""); err.Error() != "RPC interface must not be empty" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	nnet2[0].Client.RegisterPeer(nnet2[1].name, nnet2[1].Client.rpc, nnet2[1].SSLFingerprint())
+
+	if err := nnet2[0].Client.RegisterPeer(nnet2[1].name, nnet2[1].Client.rpc, ""); err.Error() != "Peer already registered: TestNode-1" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	peers, peerFps := nnet2[0].Client.Peers()
+	if res := fmt.Sprint(peers); res != "[TestNode-1]" && peerFps[0] == nnet2[1].SSLFingerprint() {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+
+	// Send the data request
+
+	datares, err = nnet2[0].Client.SendData(nnet2[1].name, map[string]string{
+		"foo": "bar",
+	}, []byte("testmsg"))
+
+	if dataReceived != "testmsg" || fmt.Sprint(ctrlReceived) != "map[foo:bar]" ||
+		string(datares) != "testack" || err != nil {
+		t.Error("Unexpected result: ", ctrlReceived, dataReceived, datares, err)
+		return
+	}
+
+	// Close connection and send data again (connection should be automatically reconnected)
+
+	nnet2[0].Client.conns[nnet2[1].name].Close()
+
+	datares, err = nnet2[0].Client.SendData(nnet2[1].name, map[string]string{
+		"foo": "bar",
+	}, []byte("testmsg"))
+
+	if dataReceived != "testmsg" || fmt.Sprint(ctrlReceived) != "map[foo:bar]" ||
+		string(datares) != "testack" || err != nil {
+		t.Error("Unexpected result: ", ctrlReceived, dataReceived, datares, err)
+		return
+	}
+
+	nnet2[0].Client.RemovePeer(nnet2[1].name)
+	nnet2[0].Client.RegisterPeer(nnet2[1].name, nnet2[1].Client.rpc, "123")
+
+	res, rfp, err = nnet2[0].Client.SendPing(nnet2[1].name, nnet2[1].Client.rpc)
+
+	if fmt.Sprint(res) != "[]" || rfp != "" || err == nil ||
+		err.Error() != "RufsError: Unexpected SSL certificate from target node (TestNode-1)" {
+		t.Error("Unexpected result:", res, rfp, err)
+		return
+	}
+
+	// Test error response
+
+	nnet2[0].Client.RemovePeer(nnet2[1].name)
+
+	_, err = nnet2[0].Client.SendData(nnet2[1].name, map[string]string{
+		"foo": "bar",
+	}, []byte("testmsg"))
+
+	if err == nil || err.Error() != "Unknown peer: TestNode-1" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	nnet2[0].Client.RegisterPeer(nnet2[1].name, nnet2[1].Client.rpc, "")
+
+	res, rfp, err = nnet2[0].Client.SendPing("", "localhost")
+
+	if err == nil || rfp != "" {
+		t.Error("Unexpected result:", res, rfp, err)
+		return
+	}
+
+	rres, err := nnet2[0].Client.SendRequest("", RPCPing, nil)
+
+	if err == nil || err.Error() != "RufsError: Unknown target node" {
+		t.Error("Unexpected result:", rres, err)
+		return
+	}
+
+	rres, err = nnet2[0].Client.SendRequest(nnet2[1].name, "foo", nil)
+
+	if err == nil || err.Error() != "RufsError: Remote error (rpc: can't find method RufsServer.foo)" {
+		t.Error("Unexpected result:", rres, err)
+		return
+	}
+
+	nnet2[1].DataHandler = func(ctrl map[string]string, data []byte) ([]byte, error) {
+		return nil, fmt.Errorf("Testerror")
+	}
+
+	datares, err = nnet2[0].Client.SendData(nnet2[1].name, nil, []byte("testmsg"))
+
+	if err == nil || err.Error() != "RufsError: Remote error (Testerror)" {
+		t.Error("Unexpected result: ", dataReceived, datares, err)
+		return
+	}
+
+	nnet2[1].DataHandler = func(ctrl map[string]string, data []byte) ([]byte, error) {
+		return nil, &Error{ErrNodeComm, "Testerror2", false}
+	}
+
+	datares, err = nnet2[0].Client.SendData(nnet2[1].name, nil, []byte("testmsg"))
+
+	if err == nil || err.Error() != "RufsError: Network error (Testerror2)" {
+		t.Error("Unexpected result: ", dataReceived, datares, err)
+		return
+	}
+}
+
+func TestNodeErrors(t *testing.T) {
+	// Debug logging
+
+	//liveOutput = true
+	//LogDebug = LogInfo
+	// defer func() { liveOutput = false }()
+
+	n := NewNode(fmt.Sprintf("localhost:%v", 9019), "TestNode", "test123", nil, nil)
+
+	n.Start(nil)
+
+	if err := n.Start(nil); err.Error() != "Cannot start node TestNode twice" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	cl := NewClient("test123", nil)
+	res, rfp, err := cl.SendPing("TestNode", fmt.Sprintf("localhost:%v", 9019))
+
+	if fmt.Sprint(res) != "[Pong]" || err != nil || rfp != "" {
+		t.Error("Unexpected result:", res, rfp, err)
+		return
+	}
+
+	// Now corrupt the token of the Client
+
+	cl.token.NodeAuth = "123"
+
+	_, _, err = cl.SendPing("TestNode", fmt.Sprintf("localhost:%v", 9019))
+
+	if err == nil || err.Error() != "RufsError: Remote error (Invalid node token)" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	cl.RegisterPeer("TestNode", fmt.Sprintf("localhost:%v", 9019), "")
+
+	_, err = cl.SendData("TestNode", nil, nil)
+
+	if err == nil || err.Error() != "RufsError: Remote error (Invalid node token)" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	n.Shutdown()
+
+	if err := n.Shutdown(); err != nil {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	n = NewNode(fmt.Sprintf("fff"), "TestNode", "test123", nil, nil)
+
+	if err := n.Start(nil); err == nil {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+}

+ 182 - 0
node/server.go

@@ -0,0 +1,182 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package node
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"crypto/sha512"
+	"fmt"
+	"net/rpc"
+
+	"devt.de/krotik/common/errorutil"
+)
+
+func init() {
+
+	// Create singleton Server instance.
+
+	rufsServer = &RufsServer{make(map[string]*RufsNode)}
+
+	// Register the cluster API as RPC server
+
+	errorutil.AssertOk(rpc.Register(rufsServer))
+}
+
+/*
+RPCFunction is used to identify the called function in a RPC call
+*/
+type RPCFunction string
+
+/*
+List of all possible RPC functions. The list includes all RPC callable functions
+in this file.
+*/
+const (
+
+	// General functions
+
+	RPCPing RPCFunction = "Ping"
+	RPCData RPCFunction = "Data"
+)
+
+/*
+RequestArgument is used to identify arguments in a RPC call
+*/
+type RequestArgument int
+
+/*
+List of all possible arguments in a RPC request. There are usually no checks which
+give back an error if a required argument is missing. The RPC API is an internal
+API and might change without backwards compatibility.
+*/
+const (
+
+	// General arguments
+
+	RequestTARGET RequestArgument = iota // Required argument which identifies the target node
+	RequestTOKEN                         // Client token which is used for authorization checks
+	RequestCTRL                          // Control object (i.e. what to do with the data)
+	RequestDATA                          // Data object
+)
+
+/*
+rufsServer is the Server instance which serves rpc calls
+*/
+var rufsServer *RufsServer
+
+/*
+RufsServer is the RPC exposed Rufs API of a machine. Server is a singleton and will
+route incoming (authenticated) requests to registered RufsNodes. The calling
+node is referred to as source node and the called node is referred to as
+target node.
+*/
+type RufsServer struct {
+	nodes map[string]*RufsNode // Map of local RufsNodes
+}
+
+// General functions
+// =================
+
+/*
+Ping answers with a Pong if the given client token was verified and the local
+node exists.
+*/
+func (s *RufsServer) Ping(request map[RequestArgument]interface{},
+	response *interface{}) error {
+
+	// Verify the given token and retrieve the target member
+
+	if _, err := s.checkToken(request); err != nil {
+		return err
+	}
+
+	// Send a simple response
+
+	res := []string{"Pong"}
+
+	*response = res
+
+	return nil
+}
+
+/*
+Data handles data requests.
+*/
+func (s *RufsServer) Data(request map[RequestArgument]interface{},
+	response *interface{}) error {
+
+	// Verify the given token and retrieve the target member
+
+	node, err := s.checkToken(request)
+
+	if err != nil || node.DataHandler == nil {
+		return err
+	}
+
+	// Forward to the registered data handler
+
+	res, err := node.DataHandler(request[RequestCTRL].(map[string]string),
+		request[RequestDATA].([]byte))
+
+	if err == nil {
+		*response = res
+	}
+
+	return err
+}
+
+// Helper functions
+// ================
+
+/*
+checkToken checks the member token in a given request.
+*/
+func (s *RufsServer) checkToken(request map[RequestArgument]interface{}) (*RufsNode, error) {
+	err := ErrUnknownTarget
+
+	// Get the target member
+
+	target := request[RequestTARGET].(string)
+	token := request[RequestTOKEN].(*RufsNodeToken)
+
+	if node, ok := s.nodes[target]; ok {
+		err = ErrInvalidToken
+
+		// Generate expected auth from given requesting node name in token and secret of target
+
+		expectedAuth := fmt.Sprintf("%X", sha512.Sum512_224([]byte(token.NodeName+node.secret)))
+
+		if token.NodeAuth == expectedAuth {
+			return node, nil
+		}
+	}
+
+	return nil, err
+}
+
+/*
+fingerprint converts a given set of bytes to a fingerprint.
+*/
+func fingerprint(b []byte) string {
+	var buf bytes.Buffer
+
+	hs := fmt.Sprintf("%x", sha256.Sum256(b))
+
+	for i, c := range hs {
+		buf.WriteByte(byte(c))
+		if (i+1)%2 == 0 && i != len(hs)-1 {
+			buf.WriteByte(byte(':'))
+		}
+	}
+
+	return buf.String()
+}

BIN
rufs.debug


File diff suppressed because it is too large
+ 1 - 0
swagger.json


+ 61 - 0
term/defs.go

@@ -0,0 +1,61 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+/*
+Package term contains a terminal implementation which can control Rufs trees.
+*/
+package term
+
+/*
+cmdMap contains all available commands
+*/
+var cmdMap = map[string]func(*TreeTerm, ...string) (string, error){
+	"?":        cmdHelp,
+	"help":     cmdHelp,
+	"cd":       cmdCd,
+	"dir":      cmdDir,
+	"ll":       cmdDir,
+	"checksum": cmdChecksum,
+	"tree":     cmdTree,
+	"branch":   cmdBranch,
+	"mount":    cmdMount,
+	"reset":    cmdReset,
+	"ping":     cmdPing,
+	"cat":      cmdCat,
+	"get":      cmdGet,
+	"put":      cmdPut,
+	"rm":       cmdRm,
+	"ren":      cmdRen,
+	"mkdir":    cmdMkDir,
+	"cp":       cmdCp,
+	"sync":     cmdSync,
+	"refresh":  cmdRefresh,
+}
+
+var helpMap = map[string]string{
+	"help [cmd]":             "Show general or command specific help",
+	"cd [path]":              "Show or change the current directory",
+	"dir [path] [glob]":      "Show a directory listing",
+	"checksum [path] [glob]": "Show a directory listing and file checksums",
+	"tree [path] [glob]":     "Show the listing of a directory and its subdirectories",
+	"branch [branch name] [rpc] [fingerprint]": "List all known branches or add a new branch to the tree",
+	"mount [path] [branch name] [ro]":          "List all mount points or add a new mount point to the tree",
+	"reset [mounts|brances]":                   "Remove all mounts or all mounts and all branches",
+	"ping <branch name> [rpc]":                 "Ping a remote branch",
+	"cat <file>":                               "Read and print the contents of a file",
+	"get <src file> [dst local file]":          "Retrieve a file and store it locally (in the current directory)",
+	"put [src local file] [dst file]":          "Read a local file and store it",
+	"rm <file>":                                "Delete a file or directory (* all files; ** all files/recursive)",
+	"ren <file> <newfile>":                     "Rename a file or directory",
+	"mkdir <dir>":                              "Create a new direectory",
+	"cp <src file/dir> <dst dir>":              "Copy a file or directory",
+	"sync <src dir> <dst dir>":                 "Make sure dst has the same files and directories as src",
+	"refresh":                                  "Refreshes all known branches and reconnect if possible.",
+}

+ 79 - 0
term/dir.go

@@ -0,0 +1,79 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"fmt"
+	"os"
+
+	"devt.de/krotik/common/stringutil"
+	"devt.de/krotik/rufs"
+)
+
+/*
+cmdCd show or change the current directory.
+*/
+func cmdCd(tt *TreeTerm, arg ...string) (string, error) {
+	if len(arg) > 0 {
+		tt.cd = tt.parsePathParam(arg[0])
+	}
+
+	return fmt.Sprint(tt.cd, "\n"), nil
+}
+
+/*
+cmdDir shows a directory listing.
+*/
+func cmdDir(tt *TreeTerm, arg ...string) (string, error) {
+	return cmdDirListing(tt, false, false, arg...)
+}
+
+/*
+cmdChecksum shows a directory listing and the checksums.
+*/
+func cmdChecksum(tt *TreeTerm, arg ...string) (string, error) {
+	return cmdDirListing(tt, false, true, arg...)
+}
+
+/*
+cmdTree shows the listing of a directory and its subdirectorie
+*/
+func cmdTree(tt *TreeTerm, arg ...string) (string, error) {
+	return cmdDirListing(tt, true, false, arg...)
+}
+
+/*
+cmdDirListing shows a directory listing and optional also its subdirectories.
+*/
+func cmdDirListing(tt *TreeTerm, recursive bool, checksum bool, arg ...string) (string, error) {
+	var dirs []string
+	var fis [][]os.FileInfo
+	var err error
+	var res, rex string
+
+	dir := tt.cd
+
+	if len(arg) > 0 {
+		dir = tt.parsePathParam(arg[0])
+
+		if len(arg) > 1 {
+			rex, err = stringutil.GlobToRegex(arg[1])
+		}
+	}
+
+	if err == nil {
+		if dirs, fis, err = tt.tree.Dir(dir, rex, recursive, checksum); err == nil {
+			res = rufs.DirResultToString(dirs, fis)
+		}
+	}
+
+	return res, err
+}

+ 230 - 0
term/file.go

@@ -0,0 +1,230 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+
+	"devt.de/krotik/common/bitutil"
+	"devt.de/krotik/rufs"
+)
+
+/*
+cmdCat reads and prints the contents of a file.
+*/
+func cmdCat(tt *TreeTerm, arg ...string) (string, error) {
+	err := fmt.Errorf("cat requires a file path")
+
+	if len(arg) > 0 {
+		err = tt.tree.ReadFileToBuffer(tt.parsePathParam(arg[0]), tt.out)
+	}
+
+	return "", err
+}
+
+/*
+cmdGet Retrieve a file and store it locally (in the current directory).
+*/
+func cmdGet(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	lenArg := len(arg)
+	err := fmt.Errorf("get requires at least a source file path")
+
+	if lenArg > 0 {
+		var f *os.File
+
+		src := tt.parsePathParam(arg[0])
+		dst := src
+
+		if lenArg > 1 {
+			dst = tt.parsePathParam(arg[1])
+		}
+
+		// Make sure we only write files to the local folder
+
+		_, dst = filepath.Split(dst)
+
+		if f, err = os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0660); err == nil {
+			defer f.Close()
+
+			if err = tt.tree.ReadFileToBuffer(src, f); err == nil {
+				res = fmt.Sprintf("Written file %s", dst)
+			}
+		}
+	}
+
+	return res, err
+}
+
+/*
+cmdPut Read a local file and store it.
+*/
+func cmdPut(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	lenArg := len(arg)
+	err := fmt.Errorf("put requires a source and destination file path")
+
+	if lenArg > 0 {
+		var f *os.File
+
+		src := arg[0]
+		dst := tt.parsePathParam(arg[1])
+
+		if f, err = os.Open(src); err == nil {
+			defer f.Close()
+
+			if err = tt.tree.WriteFileFromBuffer(dst, f); err == nil {
+				res = fmt.Sprintf("Written file %s", dst)
+			}
+		}
+	}
+
+	return res, err
+}
+
+/*
+cmdRm Delete a file or directory.
+*/
+func cmdRm(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	lenArg := len(arg)
+	err := fmt.Errorf("rm requires a file path")
+
+	if lenArg > 0 {
+
+		p := tt.parsePathParam(arg[0])
+		dir, file := path.Split(p)
+
+		if file == "" {
+
+			// If a path is give just chop off the last slash and try again
+
+			dir, file = path.Split(dir[:len(dir)-1])
+		}
+
+		_, err = tt.tree.ItemOp(dir, map[string]string{
+			rufs.ItemOpAction: rufs.ItemOpActDelete,
+			rufs.ItemOpName:   file,
+		})
+	}
+
+	return res, err
+}
+
+/*
+cmdRen Rename a file or directory.
+*/
+func cmdRen(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	lenArg := len(arg)
+	err := fmt.Errorf("ren requires a filename and a new filename")
+
+	if lenArg > 1 {
+
+		p := tt.parsePathParam(arg[0])
+		p2 := tt.parsePathParam(arg[1])
+
+		dir1, file1 := path.Split(p)
+		dir2, file2 := path.Split(p2)
+
+		if file2 == "" || dir2 != "/" {
+			err = fmt.Errorf("new filename must not have a path")
+
+		} else {
+
+			if file1 == "" {
+
+				// If a path is give just chop off the last slash and try again
+
+				dir1, file1 = path.Split(dir1[:len(dir1)-1])
+			}
+
+			_, err = tt.tree.ItemOp(dir1, map[string]string{
+				rufs.ItemOpAction:  rufs.ItemOpActRename,
+				rufs.ItemOpName:    file1,
+				rufs.ItemOpNewName: file2,
+			})
+		}
+	}
+
+	return res, err
+}
+
+/*
+cmdMkDir Create a new direectory.
+*/
+func cmdMkDir(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	lenArg := len(arg)
+	err := fmt.Errorf("mkdir requires a directory path")
+
+	if lenArg > 0 {
+
+		p := tt.parsePathParam(arg[0])
+		dir, newdir := path.Split(p)
+
+		if newdir == "" {
+
+			// If a path is given just chop off the last slash and try again
+
+			dir, newdir = path.Split(dir[:len(dir)-1])
+		}
+
+		_, err = tt.tree.ItemOp(dir, map[string]string{
+			rufs.ItemOpAction: rufs.ItemOpActMkDir,
+			rufs.ItemOpName:   newdir,
+		})
+	}
+
+	return res, err
+}
+
+/*
+cmdCp Copy a file.
+*/
+func cmdCp(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	lenArg := len(arg)
+	err := fmt.Errorf("cp requires a source file or directory and a destination directory")
+
+	if lenArg > 1 {
+
+		src := tt.parsePathParam(arg[0])
+		dst := tt.parsePathParam(arg[1])
+
+		updFunc := func(file string, writtenBytes, totalBytes, currentFile, totalFiles int64) {
+
+			if writtenBytes > 0 {
+				tt.WriteStatus(fmt.Sprintf("Copy %v: %v / %v (%v of %v)", file,
+					bitutil.ByteSizeString(writtenBytes, false),
+					bitutil.ByteSizeString(totalBytes, false),
+					currentFile, totalFiles))
+			} else {
+				tt.ClearStatus()
+			}
+		}
+
+		if err = tt.tree.Copy([]string{src}, dst, updFunc); err == nil {
+			res = "Done"
+		}
+	}
+
+	return res, err
+}

+ 560 - 0
term/file_test.go

@@ -0,0 +1,560 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/config"
+)
+
+func TestSimpleFileOperations(t *testing.T) {
+	var buf bytes.Buffer
+
+	// Build up a tree with multiple branches
+
+	cfg := map[string]interface{}{
+		config.TreeSecret: "123",
+	}
+
+	tree, _ := rufs.NewTree(cfg, clientCert)
+
+	fooRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+	fooFP := footest.SSLFingerprint()
+	barRPC := fmt.Sprintf("%v:%v", branchConfigs["bartest"][config.RPCHost], branchConfigs["bartest"][config.RPCPort])
+	barFP := bartest.SSLFingerprint()
+	tmpRPC := fmt.Sprintf("%v:%v", branchConfigs["tmptest"][config.RPCHost], branchConfigs["tmptest"][config.RPCPort])
+	tmpFP := tmptest.SSLFingerprint()
+
+	tree.AddBranch(footest.Name(), fooRPC, fooFP)
+	tree.AddBranch(bartest.Name(), barRPC, barFP)
+	tree.AddBranch(tmptest.Name(), tmpRPC, tmpFP)
+
+	tree.AddMapping("/", footest.Name(), false)
+	tree.AddMapping("/backup", bartest.Name(), true)
+
+	term := NewTreeTerm(tree, &buf)
+
+	if res, err := term.Run("tree"); err != nil || (res != `
+/
+drwxrwxrwx   0 B   backup
+drwxrwx--- 4.0 KiB sub1
+-rwxrwx---  10 B   test1
+-rwxrwx---  10 B   test2
+
+/backup
+-rwxrwx--- 10 B   test1
+
+/sub1
+-rwxrwx--- 17 B   test3
+`[1:] && res != `
+/
+drwxrwxrwx  0 B   backup
+drwxrwxrwx  0 B   sub1
+-rw-rw-rw- 10 B   test1
+-rw-rw-rw- 10 B   test2
+
+/backup
+-rw-rw-rw- 10 B   test1
+
+/sub1
+-rw-rw-rw- 17 B   test3
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	res, err := term.Run("cat /sub1/test3")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	res += buf.String()
+	res += "\n"
+	buf.Reset()
+
+	if res != `
+Sub dir test file
+`[1:] {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+
+	// Read file and store as same filename
+
+	res, err = term.Run("get /sub1/test3")
+	if err != nil || res != "Written file test3" {
+		t.Error(res, err)
+		return
+	}
+
+	defer os.Remove("test3")
+
+	content, err := ioutil.ReadFile("test3")
+	if err != nil || string(content) != "Sub dir test file" {
+		t.Error(string(content), err)
+		return
+	}
+
+	// Read file and store as same different filename - extra path is ignored
+
+	res, err = term.Run("get /sub1/test3 foo42/test123")
+	if err != nil || res != "Written file test123" {
+		t.Error(res, err)
+		return
+	}
+
+	defer os.RemoveAll("test123")
+
+	content, err = ioutil.ReadFile("test123")
+	if err != nil || string(content) != "Sub dir test file" {
+		t.Error(string(content), err)
+		return
+	}
+
+	_, err = term.Run("cat /sub1/test4")
+	if err == nil || (err.Error() != "RufsError: Remote error (stat /sub1/test4: no such file or directory)" &&
+		err.Error() != `RufsError: Remote error (CreateFile \sub1\test4: The system cannot find the file specified.)`) {
+		t.Error(err)
+		return
+	}
+
+	// Try writing files - only backup is mounted as writable
+
+	ioutil.WriteFile("testfile66", []byte("write test"), 0660)
+	defer os.Remove("testfile66")
+
+	res, err = term.Run("put testfile66 /testfile66")
+	if err == nil || err.Error() != "All applicable branches for the requested path were mounted as not writable" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("put testfile66 /backup/testfile66")
+	if err != nil || res != "Written file /backup/testfile66" {
+		t.Error(res, err)
+		return
+	}
+
+	if res, err := term.Run("tree /backup"); err != nil || (res != `
+/backup
+-rwxrwx--- 10 B   test1
+-rw-r--r-- 10 B   testfile66
+`[1:] && res != `
+/backup
+-rw-rw-rw- 10 B   test1
+-rw-rw-rw- 10 B   testfile66
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	res, err = term.Run("cat /backup/testfile66")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	res += buf.String()
+	res += "\n"
+	buf.Reset()
+
+	if res != `
+write test
+`[1:] {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+
+	if res := dirLocal("./bar"); res != `
+test1
+testfile66
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	tree.AddMapping("/backup/tmp", tmptest.Name(), true)
+
+	if res, err := term.Run("tree /backup"); err != nil || (res != `
+/backup
+-rwxrwx--- 10 B   test1
+-rw-r--r-- 10 B   testfile66
+drwxrwxrwx  0 B   tmp
+
+/backup/tmp
+`[1:] && res != `
+/backup
+-rw-rw-rw- 10 B   test1
+-rw-rw-rw- 10 B   testfile66
+drwxrwxrwx  0 B   tmp
+
+/backup/tmp
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	res, err = term.Run("put testfile66 /backup/tmp/foofile66")
+	if err != nil || res != "Written file /backup/tmp/foofile66" {
+		t.Error(res, err)
+		return
+	}
+
+	// Check that tmp has become a real directory as it was created in backup
+
+	if res, err := term.Run("tree /backup"); err != nil || (res != `
+/backup
+-rwxrwx---  10 B   test1
+-rw-r--r--  10 B   testfile66
+drwxr-xr-x 4.0 KiB tmp
+
+/backup/tmp
+-rw-r--r-- 10 B   foofile66
+`[1:] && res != `
+/backup
+-rw-rw-rw- 10 B   test1
+-rw-rw-rw- 10 B   testfile66
+drwxrwxrwx  0 B   tmp
+
+/backup/tmp
+-rw-rw-rw- 10 B   foofile66
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	// The file should have been now written to two files
+
+	if res := dirLocal("./bar"); res != `
+test1
+testfile66
+tmp(dir)
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	if res := dirLocal("./bar/tmp"); res != `
+foofile66
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	if res := dirLocal("./tmp"); res != `
+foofile66
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	// Delete one of the files
+
+	os.RemoveAll("bar/tmp")
+
+	res, err = term.Run("rm /backup/tmp/foofile66")
+	if err != nil || res != "" {
+		t.Error(res, err)
+		return
+	}
+
+	// See that the file was deleted
+
+	if res := dirLocal("./tmp"); res != `
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	// Recreate the files
+
+	res, err = term.Run("put testfile66 /backup/tmp/foofile66")
+	if err != nil || res != "Written file /backup/tmp/foofile66" {
+		t.Error(res, err)
+		return
+	}
+
+	if res := dirLocal("./bar/tmp"); res != `
+foofile66
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	if res := dirLocal("./tmp"); res != `
+foofile66
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res, err = term.Run("rm /backup/tmp1/")
+	if err == nil || err.Error() != "RufsError: Remote error (file does not exist)" {
+		t.Error("Unexpected result:", res, err)
+		return
+	}
+
+	res, err = term.Run("rm /backup/tmp1")
+	if err == nil || err.Error() != "RufsError: Remote error (file does not exist)" {
+		t.Error("Unexpected result:", res, err)
+		return
+	}
+
+	res, err = term.Run("rm /backup/*")
+	if err != nil || res != "" {
+		t.Error(res, err)
+		return
+	}
+
+	if res, err := term.Run("tree /backup"); err != nil || (res != `
+/backup
+drwxrwxrwx 0 B   tmp
+
+/backup/tmp
+-rw-r--r-- 10 B   foofile66
+`[1:] && res != `
+/backup
+drwxrwxrwx 0 B   tmp
+
+/backup/tmp
+-rw-rw-rw- 10 B   foofile66
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	// Now try to delete recursive
+
+	res, err = term.Run("rm /backup/**")
+	if err != nil || res != "" {
+		t.Error(res, err)
+		return
+	}
+
+	if res, err := term.Run("tree /backup"); err != nil || (res != `
+/backup
+drwxrwxrwx 0 B   tmp
+
+/backup/tmp
+`[1:] && res != `
+/backup
+sdrwxrwxrwx 0 KiB tmp
+
+/backup/tmp
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	// Recreate the files
+
+	res, err = term.Run("put testfile66 /backup/tmp/foofile66")
+	if err != nil || res != "Written file /backup/tmp/foofile66" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("put testfile66 /backup/foofile66")
+	if err != nil || res != "Written file /backup/foofile66" {
+		t.Error(res, err)
+		return
+	}
+
+	// Delete with wildcard
+
+	res, err = term.Run("rm /backup/foo**")
+	if err != nil || res != "" {
+		t.Error(res, err)
+		return
+	}
+
+	// Recreate files
+
+	res, err = term.Run("put testfile66 /backup/tmp/foofile66")
+	if err != nil || res != "Written file /backup/tmp/foofile66" {
+		t.Error(res, err)
+		return
+	}
+
+	if res, err := term.Run("tree /backup"); err != nil || (res != `
+/backup
+drwxr-xr-x 4.0 KiB tmp
+
+/backup/tmp
+-rw-r--r-- 10 B   foofile66
+`[1:] && res != `
+/backup
+drwxrwxrwx 0 B   tmp
+
+/backup/tmp
+-rw-rw-rw- 10 B   foofile66
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	res, err = term.Run("ren /backup/tmp")
+	if err == nil || err.Error() != "ren requires a filename and a new filename" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("ren /backup/tmp /tmp/t")
+	if err == nil || err.Error() != "new filename must not have a path" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("ren /backup/tmp tmp2")
+	if err != nil || res != "" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("ren /backup/tmp2/foofile66/ foofile67")
+	if err != nil || res != "" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("mkdir /backup/tmp2/aaa/bbb/")
+	if err != nil || res != "" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("cp /backup/tmp2/foofile67 /backup/tmp/")
+	if err != nil || res != "Done" {
+		t.Error(res, err)
+		return
+	}
+
+	if buf.String() != "\rCopy /foofile67: 10 B / 10 B (1 of 1)\r                                     \r" {
+		t.Errorf("Unexpected buffer: %#v", buf.String())
+		return
+	}
+
+	res, err = term.Run("cp /backup/tmp2/foofile68 /backup/tmp/foofile67")
+	if err == nil || err.Error() != "Cannot stat /backup/tmp2/foofile68: RufsError: Remote error (file does not exist)" {
+		t.Error(res, err)
+		return
+	}
+
+	res, err = term.Run("cp /backup/tmp2/foofile67 /foofile67")
+	if err == nil || err.Error() != "Cannot copy /backup/tmp2/foofile67 to /foofile67: All applicable branches for the requested path were mounted as not writable" {
+		t.Error(res, err)
+		return
+	}
+
+	if res, err := term.Run("tree /backup"); err != nil || (res != `
+/backup
+drwxr-xr-x 4.0 KiB tmp
+drwxr-xr-x 4.0 KiB tmp2
+
+/backup/tmp
+-rw-r--r-- 10 B   foofile66
+-rw-r--r-- 10 B   foofile67
+
+/backup/tmp2
+drwxr-xr-x 4.0 KiB aaa
+-rw-r--r--  10 B   foofile67
+
+/backup/tmp2/aaa
+drwxr-xr-x 4.0 KiB bbb
+
+/backup/tmp2/aaa/bbb
+`[1:] && res != `
+/backup
+drwxrwxrwx 0 B   tmp
+drwxrwxrwx 0 B   tmp2
+
+/backup/tmp
+-rw-rw-rw- 10 B   foofile66
+-rw-rw-rw- 10 B   foofile67
+
+/backup/tmp2
+drwxrwxrwx 0  B   aaa
+-rw-rw-rw- 10 B   foofile67
+
+/backup/tmp2/aaa
+drwxrwxrwx 0 B   bbb
+
+/backup/tmp2/aaa/bbb
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	os.Remove("./tmp/foofile66")
+	os.Remove("./tmp/foofile67")
+
+	if res := dirLocal("./tmp"); res != `
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	if res := dirLocal("./foo"); res != `
+sub1(dir)
+test1
+test2
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	if res := dirLocal("./foo/sub1"); res != `
+test3
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	os.RemoveAll("./bar/tmp")
+	os.RemoveAll("./bar/tmp2")
+	ioutil.WriteFile("bar/test1", []byte("Test3 file"), 0770)
+
+	if res := dirLocal("./bar"); res != `
+test1
+`[1:] {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+}
+
+/*
+dirLocal reads a local directory and returns all found file names as a string.
+*/
+func dirLocal(dir string) string {
+	var buf bytes.Buffer
+
+	fis, err := ioutil.ReadDir(dir)
+	errorutil.AssertOk(err)
+
+	for _, fi := range fis {
+		buf.WriteString(fi.Name())
+		if fi.IsDir() {
+			buf.WriteString("(dir)")
+		}
+		buf.WriteString(fmt.Sprintln())
+	}
+
+	return buf.String()
+}

+ 57 - 0
term/help.go

@@ -0,0 +1,57 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+	"unicode/utf8"
+
+	"devt.de/krotik/common/stringutil"
+)
+
+/*
+cmdHelp executes the help command.
+*/
+func cmdHelp(tt *TreeTerm, arg ...string) (string, error) {
+	var res bytes.Buffer
+
+	if len(arg) == 0 {
+		var maxlen = 0
+
+		cmds := make([]string, 0, len(helpMap))
+
+		res.WriteString("Available commands:\n")
+		res.WriteString("----\n")
+
+		for c := range helpMap {
+
+			if cc := utf8.RuneCountInString(c); cc > maxlen {
+				maxlen = cc
+			}
+
+			cmds = append(cmds, c)
+		}
+
+		sort.Strings(cmds)
+
+		for _, c := range cmds {
+			cc := utf8.RuneCountInString(c)
+			spacer := stringutil.GenerateRollingString(" ", maxlen-cc)
+
+			res.WriteString(fmt.Sprintf("%v%v : %v\n", c, spacer, helpMap[c]))
+
+		}
+	}
+
+	return res.String(), nil
+}

+ 98 - 0
term/mount.go

@@ -0,0 +1,98 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"bytes"
+	"fmt"
+)
+
+/*
+cmdReset removes all present mount points or branches.
+*/
+func cmdReset(tt *TreeTerm, arg ...string) (string, error) {
+
+	if len(arg) > 0 {
+		if arg[0] == "mounts" {
+			tt.tree.Reset(false)
+			return "Resetting all mounts\n", nil
+		} else if arg[0] == "branches" {
+			tt.tree.Reset(true)
+			return "Resetting all branches and mounts\n", nil
+		}
+	}
+
+	return "", fmt.Errorf("Can either reset all [mounts] or all [branches] which includes all mount points")
+}
+
+/*
+cmdBranch lists all known branches or adds a new branch to the tree.
+*/
+func cmdBranch(tt *TreeTerm, arg ...string) (string, error) {
+	var err error
+	var res bytes.Buffer
+
+	writeKnownBranches := func() {
+		braches, fps := tt.tree.ActiveBranches()
+		for i, b := range braches {
+			res.WriteString(fmt.Sprintf("%v [%v]\n", b, fps[i]))
+		}
+	}
+
+	if len(arg) == 0 {
+		writeKnownBranches()
+
+	} else if len(arg) > 1 {
+		var fp = ""
+
+		branchName := arg[0]
+		branchRPC := arg[1]
+
+		if len(arg) > 2 {
+			fp = arg[2]
+		}
+
+		err = tt.tree.AddBranch(branchName, branchRPC, fp)
+
+		writeKnownBranches()
+
+	} else {
+		err = fmt.Errorf("branch requires either no or at least 2 parameters")
+	}
+
+	return res.String(), err
+}
+
+/*
+cmdMount lists all mount points or adds a new mount point to the tree.
+*/
+func cmdMount(tt *TreeTerm, arg ...string) (string, error) {
+	var err error
+	var res bytes.Buffer
+
+	if len(arg) == 0 {
+		res.WriteString(tt.tree.String())
+
+	} else if len(arg) > 1 {
+		dir := arg[0]
+		branchName := arg[1]
+		writable := !(len(arg) > 2 && arg[2] == "ro") // Writeable unless stated otherwise
+
+		if err = tt.tree.AddMapping(dir, branchName, writable); err == nil {
+			res.WriteString(tt.tree.String())
+		}
+
+	} else {
+		err = fmt.Errorf("mount requires either 2 or no parameters")
+	}
+
+	return res.String(), err
+}

+ 53 - 0
term/sync.go

@@ -0,0 +1,53 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"fmt"
+
+	"devt.de/krotik/common/bitutil"
+)
+
+/*
+cmdSync Make sure dst has the same files and directories as src.
+*/
+func cmdSync(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	lenArg := len(arg)
+	err := fmt.Errorf("sync requires a source and a destination directory")
+
+	if lenArg > 1 {
+
+		src := tt.parsePathParam(arg[0])
+		dst := tt.parsePathParam(arg[1])
+
+		updFunc := func(op, srcFile, dstFile string, writtenBytes, totalBytes, currentFile, totalFiles int64) {
+
+			if writtenBytes > 0 {
+				tt.WriteStatus(fmt.Sprintf("%v (%v/%v) writing: %v -> %v %v / %v", op,
+					currentFile, totalFiles, srcFile, dstFile,
+					bitutil.ByteSizeString(writtenBytes, false),
+					bitutil.ByteSizeString(totalBytes, false)))
+			} else {
+				tt.ClearStatus()
+				fmt.Fprint(tt.out, fmt.Sprintln(fmt.Sprintf("%v (%v/%v) %v -> %v", op,
+					currentFile, totalFiles, srcFile, dstFile)))
+			}
+		}
+
+		if err = tt.tree.Sync(src, dst, true, updFunc); err == nil {
+			res = "Done"
+		}
+	}
+
+	return res, err
+}

+ 157 - 0
term/sync_test.go

@@ -0,0 +1,157 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/config"
+)
+
+func TestSyncOperation(t *testing.T) {
+	var buf bytes.Buffer
+
+	// Build up a tree with multiple branches
+
+	cfg := map[string]interface{}{
+		config.TreeSecret: "123",
+	}
+
+	ioutil.WriteFile("./bar/testfile66", []byte("write test"), 0660)
+	defer os.Remove("./bar/testfile66")
+
+	tree, _ := rufs.NewTree(cfg, clientCert)
+
+	fooRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+	fooFP := footest.SSLFingerprint()
+	barRPC := fmt.Sprintf("%v:%v", branchConfigs["bartest"][config.RPCHost], branchConfigs["bartest"][config.RPCPort])
+	barFP := bartest.SSLFingerprint()
+	tmpRPC := fmt.Sprintf("%v:%v", branchConfigs["tmptest"][config.RPCHost], branchConfigs["tmptest"][config.RPCPort])
+	tmpFP := tmptest.SSLFingerprint()
+
+	tree.AddBranch(footest.Name(), fooRPC, fooFP)
+	tree.AddBranch(bartest.Name(), barRPC, barFP)
+	tree.AddBranch(tmptest.Name(), tmpRPC, tmpFP)
+
+	tree.AddMapping("/1", footest.Name(), false)
+	tree.AddMapping("/1", bartest.Name(), false)
+	tree.AddMapping("/2", tmptest.Name(), true)
+
+	term := NewTreeTerm(tree, &buf)
+
+	if res, err := term.Run("tree"); err != nil || (res != `
+/
+drwxrwxrwx 0 B   1
+drwxrwxrwx 0 B   2
+
+/1
+drwxrwx--- 4.0 KiB sub1
+-rwxrwx---  10 B   test1
+-rwxrwx---  10 B   test2
+-rw-rw----  10 B   testfile66
+
+/1/sub1
+-rwxrwx--- 17 B   test3
+
+/2
+`[1:] && res != `
+/
+drwxrwxrwx 0 B   1
+drwxrwxrwx 0 B   2
+
+/1
+drwxrwxrwx 4.0 KiB sub1
+-rw-rw-rw-  10 B   test1
+-rw-rw-rw-  10 B   test2
+-rw-rw-rw-  10 B   testfile66
+
+/1/sub1
+-rw-rw-rw- 17 B   test3
+
+/2
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	res, err := term.Run("sync /1 /2")
+	if err != nil || res != "Done" {
+		t.Error(res, err)
+		return
+	}
+
+	if buf.String() != "Create directory (1/5)  -> /2/sub1\n\r"+
+		"Copy file (2/5) writing: /1/test1 -> /2/test1 10 B / 10 B\r                                                         \r"+
+		"Copy file (2/5) /1/test1 -> /2/test1\n\r"+
+		"Copy file (3/5) writing: /1/test2 -> /2/test2 10 B / 10 B\r                                                         \r"+
+		"Copy file (3/5) /1/test2 -> /2/test2\n\r"+
+		"Copy file (4/5) writing: /1/testfile66 -> /2/testfile66 10 B / 10 B\r                                                                   \r"+
+		"Copy file (4/5) /1/testfile66 -> /2/testfile66\n\r"+
+		"Copy file (5/5) writing: /1/sub1/test3 -> /2/sub1/test3 17 B / 17 B\r                                                                   \r"+
+		"Copy file (5/5) /1/sub1/test3 -> /2/sub1/test3\n" {
+		t.Errorf("Unexpected buffer: %#v", buf.String())
+		return
+	}
+
+	if res, err := term.Run("tree"); err != nil || (res != `
+/
+drwxrwxrwx 0 B   1
+drwxrwxrwx 0 B   2
+
+/1
+drwxrwx--- 4.0 KiB sub1
+-rwxrwx---  10 B   test1
+-rwxrwx---  10 B   test2
+-rw-rw----  10 B   testfile66
+
+/1/sub1
+-rwxrwx--- 17 B   test3
+
+/2
+drwxr-xr-x 4.0 KiB sub1
+-rw-r--r--  10 B   test1
+-rw-r--r--  10 B   test2
+-rw-r--r--  10 B   testfile66
+
+/2/sub1
+-rw-r--r-- 17 B   test3
+`[1:] && res != `
+/
+drwxrwxrwx 0 B   1
+drwxrwxrwx 0 B   2
+
+/1
+drwxrwxrwx 4.0 KiB sub1
+-rw-rw-rw-  10 B   test1
+-rw-rw-rw-  10 B   test2
+-rw-rw-rw-  10 B   testfile66
+
+/1/sub1
+-rw-rw-rw- 17 B   test3
+
+/2
+drwxrwxrwx 4.0 KiB sub1
+-rw-rw-rw-  10 B   test1
+-rw-rw-rw-  10 B   test2
+-rw-rw-rw-  10 B   testfile66
+
+/2/sub1
+-rw-rw-rw- 17 B   test3
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+}

+ 174 - 0
term/term.go

@@ -0,0 +1,174 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"fmt"
+	"io"
+	"path"
+	"sort"
+	"strings"
+	"unicode/utf8"
+
+	"devt.de/krotik/common/stringutil"
+	"devt.de/krotik/rufs"
+)
+
+/*
+TreeTerm models a command processor for Rufs trees.
+*/
+type TreeTerm struct {
+	tree       *rufs.Tree // Tree which we operate on
+	cd         string     // Current directory
+	out        io.Writer  // Output writer
+	lastStatus string     // Last status line
+}
+
+/*
+NewTreeTerm returns a new command processor for Rufs trees.
+*/
+func NewTreeTerm(t *rufs.Tree, out io.Writer) *TreeTerm {
+	return &TreeTerm{t, "/", out, ""}
+}
+
+/*
+WriteStatus writes a status line to the output writer.
+*/
+func (tt *TreeTerm) WriteStatus(line string) {
+	fmt.Fprint(tt.out, "\r")
+	fmt.Fprint(tt.out, line)
+
+	ll := len(tt.lastStatus)
+	lc := len(line)
+
+	if ll > lc {
+		fmt.Fprint(tt.out, stringutil.GenerateRollingString(" ", ll-lc))
+	}
+
+	tt.lastStatus = line
+}
+
+/*
+ClearStatus removes the last status line and returns the cursor to the initial position.
+*/
+func (tt *TreeTerm) ClearStatus() {
+	if tt.lastStatus != "" {
+		toClear := utf8.RuneCountInString(tt.lastStatus)
+		fmt.Fprint(tt.out, "\r")
+		fmt.Fprint(tt.out, stringutil.GenerateRollingString(" ", toClear))
+		fmt.Fprint(tt.out, "\r")
+	}
+}
+
+/*
+CurrentDir returns the current directory of this TreeTerm.
+*/
+func (tt *TreeTerm) CurrentDir() string {
+	return tt.cd
+}
+
+/*
+AddCmd adds a new command to the terminal
+*/
+func (tt *TreeTerm) AddCmd(cmd, helpusage, help string,
+	cmdFunc func(*TreeTerm, ...string) (string, error)) {
+
+	cmdMap[cmd] = cmdFunc
+	helpMap[helpusage] = help
+}
+
+/*
+Cmds returns a list of available terminal commands.
+*/
+func (tt *TreeTerm) Cmds() []string {
+	var cmds []string
+
+	for k := range cmdMap {
+		cmds = append(cmds, k)
+	}
+
+	sort.Strings(cmds)
+
+	return cmds
+}
+
+/*
+Run executes a given command line. And return its output as a string. File
+output and other streams to the console are written to the output writer.
+*/
+func (tt *TreeTerm) Run(line string) (string, error) {
+	var err error
+	var res string
+	var arg []string
+
+	// Parse the input
+
+	c := strings.Split(line, " ")
+
+	cmd := c[0]
+
+	if len(c) > 1 {
+		arg = c[1:]
+	}
+
+	// Execute the given command
+
+	if f, ok := cmdMap[cmd]; ok {
+		res, err = f(tt, arg...)
+	} else {
+		err = fmt.Errorf("Unknown command: %s", cmd)
+	}
+
+	return res, err
+}
+
+/*
+cmdPing pings a remote branch.
+*/
+func cmdPing(tt *TreeTerm, arg ...string) (string, error) {
+	var res string
+
+	err := fmt.Errorf("ping requires at least a branch name")
+
+	if len(arg) > 0 {
+		var fp, rpc string
+
+		if len(arg) > 1 {
+			rpc = arg[1]
+		}
+
+		if fp, err = tt.tree.PingBranch(arg[0], rpc); err == nil {
+			res = fmt.Sprint("Response ok - fingerprint: ", fp, "\n")
+		}
+	}
+
+	return res, err
+}
+
+/*
+cmdRefresh refreshes all known branches and connects depending on if the
+branches are reachable.
+*/
+func cmdRefresh(tt *TreeTerm, arg ...string) (string, error) {
+	tt.tree.Refresh()
+
+	return "Done", nil
+}
+
+/*
+parsePathParam parse a given path parameter and return an absolute path.
+*/
+func (tt *TreeTerm) parsePathParam(p string) string {
+	if !strings.HasPrefix(p, "/") {
+		p = path.Join(tt.cd, p) // Take care of relative paths
+	}
+	return p
+}

+ 479 - 0
term/term_test.go

@@ -0,0 +1,479 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package term
+
+import (
+	"bytes"
+	"crypto/tls"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/fileutil"
+	"devt.de/krotik/rufs"
+	"devt.de/krotik/rufs/config"
+)
+
+func TestStatusUpdate(t *testing.T) {
+	var buf bytes.Buffer
+
+	// Build up a tree from one branch
+
+	cfg := map[string]interface{}{
+		config.TreeSecret: "123",
+	}
+
+	tree, _ := rufs.NewTree(cfg, clientCert)
+
+	term := NewTreeTerm(tree, &buf)
+
+	term.WriteStatus("Test")
+	term.WriteStatus("foo")
+	term.ClearStatus()
+
+	// Check that we only clear extra characters when overwriting the status
+
+	if buf.String() != "\rTest\rfoo \r   \r" {
+		t.Errorf("Unexpected result: %#v", buf.String())
+		return
+	}
+}
+
+func TestSimpleTreeExploring(t *testing.T) {
+
+	// Build up a tree from one branch
+
+	cfg := map[string]interface{}{
+		config.TreeSecret: "123",
+	}
+
+	tree, _ := rufs.NewTree(cfg, clientCert)
+
+	fooRPC := fmt.Sprintf("%v:%v", branchConfigs["footest"][config.RPCHost], branchConfigs["footest"][config.RPCPort])
+	fooFP := footest.SSLFingerprint()
+
+	term := NewTreeTerm(tree, nil)
+
+	term.AddCmd("unittest", "unittest [bla]", "Unit test command", func(*TreeTerm, ...string) (string, error) {
+		return "123", nil
+	})
+
+	if res, err := term.Run("?"); err != nil || res != `
+Available commands:
+----
+branch [branch name] [rpc] [fingerprint] : List all known branches or add a new branch to the tree
+cat <file>                               : Read and print the contents of a file
+cd [path]                                : Show or change the current directory
+checksum [path] [glob]                   : Show a directory listing and file checksums
+cp <src file/dir> <dst dir>              : Copy a file or directory
+dir [path] [glob]                        : Show a directory listing
+get <src file> [dst local file]          : Retrieve a file and store it locally (in the current directory)
+help [cmd]                               : Show general or command specific help
+mkdir <dir>                              : Create a new direectory
+mount [path] [branch name] [ro]          : List all mount points or add a new mount point to the tree
+ping <branch name> [rpc]                 : Ping a remote branch
+put [src local file] [dst file]          : Read a local file and store it
+refresh                                  : Refreshes all known branches and reconnect if possible.
+ren <file> <newfile>                     : Rename a file or directory
+reset [mounts|brances]                   : Remove all mounts or all mounts and all branches
+rm <file>                                : Delete a file or directory (* all files; ** all files/recursive)
+sync <src dir> <dst dir>                 : Make sure dst has the same files and directories as src
+tree [path] [glob]                       : Show the listing of a directory and its subdirectories
+unittest [bla]                           : Unit test command
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res := term.Cmds(); fmt.Sprint(res) != "[? branch cat cd checksum cp dir "+
+		"get help ll mkdir mount ping put refresh ren reset rm sync tree unittest]" {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	if res, err := term.Run("ll"); err != nil || res != `
+/
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("mount"); err != nil || res != `
+/: 
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("branch"); err != nil || res != `
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("refresh"); err != nil || res != `
+Done`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if _, err := term.Run(fmt.Sprintf("branch myfoo %v %v", fooRPC, fooFP)); err == nil ||
+		err.Error() != "RufsError: Remote error (Unknown target node)" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	if res, err := term.Run(fmt.Sprintf("ping footest %v %v", fooRPC, "")); err != nil || res != `
+Response ok - fingerprint: `[1:]+fooFP+`
+` {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run(fmt.Sprintf("branch footest %v %v", fooRPC, "")); err != nil || res != `
+footest [`[1:]+fooFP+`]
+` {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("branch"); err != nil || res != `
+footest [`[1:]+fooFP+`]
+` {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if _, err := term.Run("mount / myfoo"); err == nil ||
+		err.Error() != "Unknown target node" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	// Mount the first directory
+
+	if _, err := term.Run("mount / footest"); err != nil {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	// The directory listing should now return something
+
+	if res, err := term.Run("ll"); err != nil || (res != `
+/
+drwxrwx--- 4.0 KiB sub1
+-rwxrwx---  10 B   test1
+-rwxrwx---  10 B   test2
+`[1:] && res != `
+/
+drwxrwxrwx  0 B   sub1
+-rw-rw-rw- 10 B   test1
+-rw-rw-rw- 10 B   test2
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("dir sub1"); err != nil || (res != `
+/sub1
+-rwxrwx--- 17 B   test3
+`[1:] && res != `
+/sub1
+-rw-rw-rw- 17 B   test3
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("dir . {"); err == nil || err.Error() != "Unclosed group at 1 of {" {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("checksum"); err != nil || (res != `
+/
+drwxrwx--- 4.0 KiB sub1
+-rwxrwx---  10 B   test1 [73b8af47]
+-rwxrwx---  10 B   test2 [b0c1fadd]
+`[1:] && res != `
+/
+drwxrwxrwx  0 B   sub1
+-rw-rw-rw- 10 B   test1 [73b8af47]
+-rw-rw-rw- 10 B   test2 [b0c1fadd]
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("tree"); err != nil || (res != `
+/
+drwxrwx--- 4.0 KiB sub1
+-rwxrwx---  10 B   test1
+-rwxrwx---  10 B   test2
+
+/sub1
+-rwxrwx--- 17 B   test3
+`[1:] && res != `
+/
+drwxrwxrwx  0 B   sub1
+-rw-rw-rw- 10 B   test1
+-rw-rw-rw- 10 B   test2
+
+/sub1
+-rw-rw-rw- 17 B   test3
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res := term.CurrentDir(); res != "/" {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+
+	if res, err := term.Run("cd sub1"); err != nil || res != `
+/sub1
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res := term.CurrentDir(); res != "/sub1" {
+		t.Error("Unexpected result: ", res)
+		return
+	}
+
+	if res, err := term.Run("checksum"); err != nil || (res != `
+/sub1
+-rwxrwx--- 17 B   test3 [f89782b1]
+`[1:] && res != `
+/sub1
+-rw-rw-rw- 17 B   test3 [f89782b1]
+`[1:]) {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if _, err := term.Run("reset"); err == nil ||
+		err.Error() != "Can either reset all [mounts] or all [branches] which includes all mount points" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	if res, err := term.Run("reset mounts"); err != nil || res != `
+Resetting all mounts
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("mount"); err != nil || res != `
+/: 
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("branch"); err != nil || res != `
+footest [`[1:]+fooFP+`]
+` {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("reset branches"); err != nil || res != `
+Resetting all branches and mounts
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	if res, err := term.Run("branch"); err != nil || res != `
+`[1:] {
+		t.Error("Unexpected result: ", res, err)
+		return
+	}
+
+	// Error returns
+
+	if _, err := term.Run("mount x"); err == nil || err.Error() != "mount requires either 2 or no parameters" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	if _, err := term.Run("branch x"); err == nil || err.Error() != "branch requires either no or at least 2 parameters" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+
+	if _, err := term.Run("bla"); err == nil || err.Error() != "Unknown command: bla" {
+		t.Error("Unexpected result: ", err)
+		return
+	}
+}
+
+const certdir = "certs" // Directory for certificates
+var portCount = 0       // Port assignment counter for Branch ports
+
+var footest, bartest, tmptest *rufs.Branch              // Branches
+var branchConfigs = map[string]map[string]interface{}{} // All branch configs
+var clientCert *tls.Certificate
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// Create a ssl certificate directory
+
+	if res, _ := fileutil.PathExists(certdir); res {
+		os.RemoveAll(certdir)
+	}
+
+	err := os.Mkdir(certdir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create client certificate
+
+	certFile := fmt.Sprintf("cert-client.pem")
+	keyFile := fmt.Sprintf("key-client.pem")
+	host := "localhost"
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(path.Join(certdir, certFile), path.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	clientCert = &cert
+
+	// Ensure logging is discarded
+
+	log.SetOutput(ioutil.Discard)
+
+	// Set up test branches
+
+	b1, err := createBranch("footest", "foo")
+	errorutil.AssertOk(err)
+
+	b2, err := createBranch("bartest", "bar")
+	errorutil.AssertOk(err)
+
+	b3, err := createBranch("tmptest", "tmp")
+	errorutil.AssertOk(err)
+
+	footest = b1
+	bartest = b2
+	tmptest = b3
+
+	// Create some test files
+
+	ioutil.WriteFile("foo/test1", []byte("Test1 file"), 0770)
+	ioutil.WriteFile("foo/test2", []byte("Test2 file"), 0770)
+
+	os.Mkdir("foo/sub1", 0770)
+	ioutil.WriteFile("foo/sub1/test3", []byte("Sub dir test file"), 0770)
+
+	ioutil.WriteFile("bar/test1", []byte("Test3 file"), 0770)
+
+	// Run the tests
+
+	res := m.Run()
+
+	// Shutdown the branches
+
+	errorutil.AssertOk(b1.Shutdown())
+	errorutil.AssertOk(b2.Shutdown())
+	errorutil.AssertOk(b3.Shutdown())
+
+	// Remove all directories again
+
+	if err = os.RemoveAll(certdir); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("foo"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("bar"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+	if err = os.RemoveAll("tmp"); err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+
+	os.Exit(res)
+}
+
+/*
+createBranch creates a new branch.
+*/
+func createBranch(name, dir string) (*rufs.Branch, error) {
+
+	// Create the path directory
+
+	if res, _ := fileutil.PathExists(dir); res {
+		os.RemoveAll(dir)
+	}
+
+	err := os.Mkdir(dir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Create the certificate
+
+	portCount++
+	host := fmt.Sprintf("localhost:%v", 9020+portCount)
+
+	// Generate a certificate and private key
+
+	certFile := fmt.Sprintf("cert-%v.pem", portCount)
+	keyFile := fmt.Sprintf("key-%v.pem", portCount)
+
+	err = cryptutil.GenCert(certdir, certFile, keyFile, host, "", 365*24*time.Hour, true, 2048, "")
+	if err != nil {
+		panic(err)
+	}
+
+	cert, err := tls.LoadX509KeyPair(filepath.Join(certdir, certFile), filepath.Join(certdir, keyFile))
+	if err != nil {
+		panic(err)
+	}
+
+	// Create the Branch
+
+	config := map[string]interface{}{
+		config.BranchName:     name,
+		config.BranchSecret:   "123",
+		config.EnableReadOnly: false,
+		config.RPCHost:        "localhost",
+		config.RPCPort:        fmt.Sprint(9020 + portCount),
+		config.LocalFolder:    dir,
+	}
+
+	branchConfigs[name] = config
+
+	return rufs.NewBranch(config, &cert)
+}

File diff suppressed because it is too large
+ 1356 - 0
tree.go


+ 138 - 0
tree_item.go

@@ -0,0 +1,138 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package rufs
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+	"sort"
+
+	"devt.de/krotik/common/errorutil"
+	"devt.de/krotik/common/stringutil"
+)
+
+/*
+treeItem models an item in the tree. This is an internal data structure
+which is not exposed.
+*/
+type treeItem struct {
+	children            map[string]*treeItem // Mapping from path component to branch
+	remoteBranches      []string             // List of remote branches which are present on this level
+	remoteBranchWriting []bool               // Flag if the remote branch should receive write requests
+}
+
+/*
+findPathBranches finds all relevant branches for a single path. The iterator
+function receives 4 parameters: The tree item, the total path within the tree,
+the subpath within the branch and a list of all branches for the tree path.
+Calling code should always give a treePath of "/".
+*/
+func (t *treeItem) findPathBranches(treePath string, branchPath []string,
+	recursive bool, visit func(*treeItem, string, []string, []string, []bool)) {
+
+	visit(t, treePath, branchPath, t.remoteBranches, t.remoteBranchWriting)
+
+	if len(branchPath) > 0 {
+
+		if c, ok := t.children[branchPath[0]]; ok {
+
+			// Check if a subpath matches
+
+			c.findPathBranches(path.Join(treePath, branchPath[0]),
+				branchPath[1:], recursive, visit)
+		}
+
+	} else if recursive {
+		var childNames []string
+
+		for n := range t.children {
+			childNames = append(childNames, n)
+		}
+
+		sort.Strings(childNames)
+
+		for _, n := range childNames {
+			t.children[n].findPathBranches(path.Join(treePath, n),
+				branchPath, recursive, visit)
+		}
+	}
+}
+
+/*
+addMapping adds a new mapping.
+*/
+func (t *treeItem) addMapping(mappingPath []string, branchName string, writable bool) {
+
+	// Add mapping to a child
+
+	if len(mappingPath) > 0 {
+
+		childName := mappingPath[0]
+		rest := mappingPath[1:]
+
+		errorutil.AssertTrue(childName != "",
+			"Adding a mapping with an empty path is not supported")
+
+		// Ensure child exists
+
+		child, ok := t.children[childName]
+		if !ok {
+			child = &treeItem{make(map[string]*treeItem), []string{}, []bool{}}
+			t.children[childName] = child
+		}
+
+		// Add rest of the mapping to the child
+
+		child.addMapping(rest, branchName, writable)
+
+		return
+	}
+
+	// Add branch name to this branch - keep the order in which the branches were added
+
+	t.remoteBranches = append(t.remoteBranches, branchName)
+	t.remoteBranchWriting = append(t.remoteBranchWriting, writable)
+}
+
+/*
+String returns a string representation of this item and its children.
+*/
+func (t *treeItem) String(indent int, buf *bytes.Buffer) {
+
+	for i, b := range t.remoteBranches {
+		buf.WriteString(b)
+
+		if t.remoteBranchWriting[i] {
+			buf.WriteString("(w)")
+		} else {
+			buf.WriteString("(r)")
+		}
+
+		if i < len(t.remoteBranches)-1 {
+			buf.WriteString(", ")
+		}
+	}
+	buf.WriteString("\n")
+
+	names := make([]string, 0, len(t.children))
+	for n := range t.children {
+		names = append(names, n)
+	}
+	sort.Strings(names)
+
+	for _, n := range names {
+		i := t.children[n]
+		buf.WriteString(stringutil.GenerateRollingString(" ", indent*2))
+		buf.WriteString(fmt.Sprintf("%v/: ", n))
+		i.String(indent+1, buf)
+	}
+}

File diff suppressed because it is too large
+ 1719 - 0
tree_test.go


+ 33 - 0
util.go

@@ -0,0 +1,33 @@
+/*
+ * Rufs - Remote Union File System
+ *
+ * Copyright 2017 Matthias Ladkau. All rights reserved.
+ *
+ * This Source Code Form is subject to the terms of the MIT
+ * License, If a copy of the MIT License was not distributed with this
+ * file, You can obtain one at https://opensource.org/licenses/MIT.
+ */
+
+package rufs
+
+import (
+	"io"
+
+	"devt.de/krotik/rufs/node"
+)
+
+/*
+IsEOF tests if the given error is an EOF error.
+*/
+func IsEOF(err error) bool {
+
+	if err == io.EOF {
+		return true
+	}
+
+	if rerr, ok := err.(*node.Error); ok {
+		return rerr.Detail == io.EOF.Error()
+	}
+
+	return false
+}

BIN
web.zip