Browse Source

Initial commit - EliasDB version 1.0.0

Matthias Ladkau 10 months ago
commit
b2e9cd9a8a
100 changed files with 19747 additions and 0 deletions
  1. 373 0
      LICENSE
  2. 168 0
      README.md
  3. 81 0
      doc/elias_db_design.md
  4. 308 0
      doc/embedding.md
  5. 130 0
      doc/eql.md
  6. 1 0
      doc/swagger.json
  7. 78 0
      examples/tutorial/doc/tutorial.md
  8. BIN
      examples/tutorial/doc/tutorial1.png
  9. BIN
      examples/tutorial/doc/tutorial2.png
  10. BIN
      examples/tutorial/doc/tutorial3.png
  11. 11 0
      examples/tutorial/start.sh
  12. BIN
      examples/tutorial/tutorial_data.zip
  13. 121 0
      src/devt.de/common/bitutil/bitutil.go
  14. 81 0
      src/devt.de/common/bitutil/bitutil_test.go
  15. 89 0
      src/devt.de/common/bitutil/murmurhash3.go
  16. 75 0
      src/devt.de/common/bitutil/murmurhash3_test.go
  17. 416 0
      src/devt.de/common/bitutil/packedlist.go
  18. 360 0
      src/devt.de/common/bitutil/packedlist_test.go
  19. 188 0
      src/devt.de/common/cryptutil/gencert.go
  20. 149 0
      src/devt.de/common/cryptutil/gencert_test.go
  21. 109 0
      src/devt.de/common/cryptutil/stringcrypt.go
  22. 53 0
      src/devt.de/common/cryptutil/stringcrypt_test.go
  23. 37 0
      src/devt.de/common/cryptutil/uuid.go
  24. 22 0
      src/devt.de/common/cryptutil/uuid_test.go
  25. 109 0
      src/devt.de/common/cryptutil/x509util.go
  26. 94 0
      src/devt.de/common/cryptutil/x509util_test.go
  27. 65 0
      src/devt.de/common/datautil/datacopy.go
  28. 80 0
      src/devt.de/common/datautil/datacopy_test.go
  29. 246 0
      src/devt.de/common/datautil/mapcache.go
  30. 148 0
      src/devt.de/common/datautil/mapcache_test.go
  31. 52 0
      src/devt.de/common/datautil/nesting.go
  32. 79 0
      src/devt.de/common/datautil/nesting_test.go
  33. 102 0
      src/devt.de/common/datautil/nonce.go
  34. 55 0
      src/devt.de/common/datautil/nonce_test.go
  35. 112 0
      src/devt.de/common/datautil/persistentmap.go
  36. 146 0
      src/devt.de/common/datautil/persistentmap_test.go
  37. 181 0
      src/devt.de/common/datautil/ringbuffer.go
  38. 115 0
      src/devt.de/common/datautil/ringbuffer_test.go
  39. 701 0
      src/devt.de/common/datautil/userdb.go
  40. 484 0
      src/devt.de/common/datautil/userdb_test.go
  41. 30 0
      src/devt.de/common/defs/rambazamba/eventsource.go
  42. 40 0
      src/devt.de/common/defs/rumble/func.go
  43. 45 0
      src/devt.de/common/defs/rumble/globals.go
  44. 22 0
      src/devt.de/common/defs/rumble/runtime.go
  45. 27 0
      src/devt.de/common/defs/rumble/variables.go
  46. 68 0
      src/devt.de/common/errorutil/errorutil.go
  47. 64 0
      src/devt.de/common/errorutil/errorutil_test.go
  48. 418 0
      src/devt.de/common/fileutil/config.go
  49. 221 0
      src/devt.de/common/fileutil/config_test.go
  50. 134 0
      src/devt.de/common/fileutil/fileutil.go
  51. 135 0
      src/devt.de/common/fileutil/fileutil_test.go
  52. 439 0
      src/devt.de/common/fileutil/multifilebuffer.go
  53. 388 0
      src/devt.de/common/fileutil/multifilebuffer_test.go
  54. 89 0
      src/devt.de/common/fileutil/zip.go
  55. 79 0
      src/devt.de/common/fileutil/zip_test.go
  56. 127 0
      src/devt.de/common/flowutil/eventpump.go
  57. 238 0
      src/devt.de/common/flowutil/eventpump_test.go
  58. 1284 0
      src/devt.de/common/httputil/access/acl.go
  59. 1271 0
      src/devt.de/common/httputil/access/acl_test.go
  60. 45 0
      src/devt.de/common/httputil/auth/auth.go
  61. 194 0
      src/devt.de/common/httputil/auth/auth_test.go
  62. 139 0
      src/devt.de/common/httputil/auth/basic.go
  63. 222 0
      src/devt.de/common/httputil/auth/basic_test.go
  64. 300 0
      src/devt.de/common/httputil/auth/cookie.go
  65. 243 0
      src/devt.de/common/httputil/auth/cookie_test.go
  66. 278 0
      src/devt.de/common/httputil/httpserver.go
  67. 290 0
      src/devt.de/common/httputil/httpserver_test.go
  68. 113 0
      src/devt.de/common/httputil/user/session.go
  69. 151 0
      src/devt.de/common/httputil/user/session_test.go
  70. 258 0
      src/devt.de/common/httputil/user/user.go
  71. 186 0
      src/devt.de/common/httputil/user/user_test.go
  72. 114 0
      src/devt.de/common/httputil/util.go
  73. 144 0
      src/devt.de/common/httputil/util_test.go
  74. 141 0
      src/devt.de/common/imageutil/asciiraster.go
  75. 163 0
      src/devt.de/common/imageutil/asciiraster_test.go
  76. 314 0
      src/devt.de/common/imageutil/rasterfont1.go
  77. 852 0
      src/devt.de/common/imageutil/rasterfont2.go
  78. 227 0
      src/devt.de/common/lockutil/lockfile.go
  79. 139 0
      src/devt.de/common/lockutil/lockfile_test.go
  80. 111 0
      src/devt.de/common/logutil/formatter.go
  81. 72 0
      src/devt.de/common/logutil/formatter_test.go
  82. 299 0
      src/devt.de/common/logutil/logger.go
  83. 179 0
      src/devt.de/common/logutil/logger_test.go
  84. 34 0
      src/devt.de/common/pools/pools.go
  85. 56 0
      src/devt.de/common/pools/pools_test.go
  86. 514 0
      src/devt.de/common/pools/threadpool.go
  87. 415 0
      src/devt.de/common/pools/threadpool_test.go
  88. 83 0
      src/devt.de/common/sortutil/heap.go
  89. 101 0
      src/devt.de/common/sortutil/heap_test.go
  90. 227 0
      src/devt.de/common/sortutil/priorityqueue.go
  91. 196 0
      src/devt.de/common/sortutil/priorityqueue_test.go
  92. 62 0
      src/devt.de/common/sortutil/sortutil.go
  93. 49 0
      src/devt.de/common/sortutil/sortutil_test.go
  94. 118 0
      src/devt.de/common/sortutil/vectorclock.go
  95. 97 0
      src/devt.de/common/sortutil/vectorclock_test.go
  96. 771 0
      src/devt.de/common/stringutil/stringutil.go
  97. 630 0
      src/devt.de/common/stringutil/stringutil_test.go
  98. 246 0
      src/devt.de/common/termutil/autoterm.go
  99. 236 0
      src/devt.de/common/termutil/autoterm_test.go
  100. 0 0
      src/devt.de/common/termutil/fileterm.go

+ 373 - 0
LICENSE

@@ -0,0 +1,373 @@
+Mozilla Public License Version 2.0
+==================================
+
+1. Definitions
+--------------
+
+1.1. "Contributor"
+    means each individual or legal entity that creates, contributes to
+    the creation of, or owns Covered Software.
+
+1.2. "Contributor Version"
+    means the combination of the Contributions of others (if any) used
+    by a Contributor and that particular Contributor's Contribution.
+
+1.3. "Contribution"
+    means Covered Software of a particular Contributor.
+
+1.4. "Covered Software"
+    means Source Code Form to which the initial Contributor has attached
+    the notice in Exhibit A, the Executable Form of such Source Code
+    Form, and Modifications of such Source Code Form, in each case
+    including portions thereof.
+
+1.5. "Incompatible With Secondary Licenses"
+    means
+
+    (a) that the initial Contributor has attached the notice described
+        in Exhibit B to the Covered Software; or
+
+    (b) that the Covered Software was made available under the terms of
+        version 1.1 or earlier of the License, but not also under the
+        terms of a Secondary License.
+
+1.6. "Executable Form"
+    means any form of the work other than Source Code Form.
+
+1.7. "Larger Work"
+    means a work that combines Covered Software with other material, in 
+    a separate file or files, that is not Covered Software.
+
+1.8. "License"
+    means this document.
+
+1.9. "Licensable"
+    means having the right to grant, to the maximum extent possible,
+    whether at the time of the initial grant or subsequently, any and
+    all of the rights conveyed by this License.
+
+1.10. "Modifications"
+    means any of the following:
+
+    (a) any file in Source Code Form that results from an addition to,
+        deletion from, or modification of the contents of Covered
+        Software; or
+
+    (b) any new file in Source Code Form that contains any Covered
+        Software.
+
+1.11. "Patent Claims" of a Contributor
+    means any patent claim(s), including without limitation, method,
+    process, and apparatus claims, in any patent Licensable by such
+    Contributor that would be infringed, but for the grant of the
+    License, by the making, using, selling, offering for sale, having
+    made, import, or transfer of either its Contributions or its
+    Contributor Version.
+
+1.12. "Secondary License"
+    means either the GNU General Public License, Version 2.0, the GNU
+    Lesser General Public License, Version 2.1, the GNU Affero General
+    Public License, Version 3.0, or any later versions of those
+    licenses.
+
+1.13. "Source Code Form"
+    means the form of the work preferred for making modifications.
+
+1.14. "You" (or "Your")
+    means an individual or a legal entity exercising rights under this
+    License. For legal entities, "You" includes any entity that
+    controls, is controlled by, or is under common control with You. For
+    purposes of this definition, "control" means (a) the power, direct
+    or indirect, to cause the direction or management of such entity,
+    whether by contract or otherwise, or (b) ownership of more than
+    fifty percent (50%) of the outstanding shares or beneficial
+    ownership of such entity.
+
+2. License Grants and Conditions
+--------------------------------
+
+2.1. Grants
+
+Each Contributor hereby grants You a world-wide, royalty-free,
+non-exclusive license:
+
+(a) under intellectual property rights (other than patent or trademark)
+    Licensable by such Contributor to use, reproduce, make available,
+    modify, display, perform, distribute, and otherwise exploit its
+    Contributions, either on an unmodified basis, with Modifications, or
+    as part of a Larger Work; and
+
+(b) under Patent Claims of such Contributor to make, use, sell, offer
+    for sale, have made, import, and otherwise transfer either its
+    Contributions or its Contributor Version.
+
+2.2. Effective Date
+
+The licenses granted in Section 2.1 with respect to any Contribution
+become effective for each Contribution on the date the Contributor first
+distributes such Contribution.
+
+2.3. Limitations on Grant Scope
+
+The licenses granted in this Section 2 are the only rights granted under
+this License. No additional rights or licenses will be implied from the
+distribution or licensing of Covered Software under this License.
+Notwithstanding Section 2.1(b) above, no patent license is granted by a
+Contributor:
+
+(a) for any code that a Contributor has removed from Covered Software;
+    or
+
+(b) for infringements caused by: (i) Your and any other third party's
+    modifications of Covered Software, or (ii) the combination of its
+    Contributions with other software (except as part of its Contributor
+    Version); or
+
+(c) under Patent Claims infringed by Covered Software in the absence of
+    its Contributions.
+
+This License does not grant any rights in the trademarks, service marks,
+or logos of any Contributor (except as may be necessary to comply with
+the notice requirements in Section 3.4).
+
+2.4. Subsequent Licenses
+
+No Contributor makes additional grants as a result of Your choice to
+distribute the Covered Software under a subsequent version of this
+License (see Section 10.2) or under the terms of a Secondary License (if
+permitted under the terms of Section 3.3).
+
+2.5. Representation
+
+Each Contributor represents that the Contributor believes its
+Contributions are its original creation(s) or it has sufficient rights
+to grant the rights to its Contributions conveyed by this License.
+
+2.6. Fair Use
+
+This License is not intended to limit any rights You have under
+applicable copyright doctrines of fair use, fair dealing, or other
+equivalents.
+
+2.7. Conditions
+
+Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted
+in Section 2.1.
+
+3. Responsibilities
+-------------------
+
+3.1. Distribution of Source Form
+
+All distribution of Covered Software in Source Code Form, including any
+Modifications that You create or to which You contribute, must be under
+the terms of this License. You must inform recipients that the Source
+Code Form of the Covered Software is governed by the terms of this
+License, and how they can obtain a copy of this License. You may not
+attempt to alter or restrict the recipients' rights in the Source Code
+Form.
+
+3.2. Distribution of Executable Form
+
+If You distribute Covered Software in Executable Form then:
+
+(a) such Covered Software must also be made available in Source Code
+    Form, as described in Section 3.1, and You must inform recipients of
+    the Executable Form how they can obtain a copy of such Source Code
+    Form by reasonable means in a timely manner, at a charge no more
+    than the cost of distribution to the recipient; and
+
+(b) You may distribute such Executable Form under the terms of this
+    License, or sublicense it under different terms, provided that the
+    license for the Executable Form does not attempt to limit or alter
+    the recipients' rights in the Source Code Form under this License.
+
+3.3. Distribution of a Larger Work
+
+You may create and distribute a Larger Work under terms of Your choice,
+provided that You also comply with the requirements of this License for
+the Covered Software. If the Larger Work is a combination of Covered
+Software with a work governed by one or more Secondary Licenses, and the
+Covered Software is not Incompatible With Secondary Licenses, this
+License permits You to additionally distribute such Covered Software
+under the terms of such Secondary License(s), so that the recipient of
+the Larger Work may, at their option, further distribute the Covered
+Software under the terms of either this License or such Secondary
+License(s).
+
+3.4. Notices
+
+You may not remove or alter the substance of any license notices
+(including copyright notices, patent notices, disclaimers of warranty,
+or limitations of liability) contained within the Source Code Form of
+the Covered Software, except that You may alter any license notices to
+the extent required to remedy known factual inaccuracies.
+
+3.5. Application of Additional Terms
+
+You may choose to offer, and to charge a fee for, warranty, support,
+indemnity or liability obligations to one or more recipients of Covered
+Software. However, You may do so only on Your own behalf, and not on
+behalf of any Contributor. You must make it absolutely clear that any
+such warranty, support, indemnity, or liability obligation is offered by
+You alone, and You hereby agree to indemnify every Contributor for any
+liability incurred by such Contributor as a result of warranty, support,
+indemnity or liability terms You offer. You may include additional
+disclaimers of warranty and limitations of liability specific to any
+jurisdiction.
+
+4. Inability to Comply Due to Statute or Regulation
+---------------------------------------------------
+
+If it is impossible for You to comply with any of the terms of this
+License with respect to some or all of the Covered Software due to
+statute, judicial order, or regulation then You must: (a) comply with
+the terms of this License to the maximum extent possible; and (b)
+describe the limitations and the code they affect. Such description must
+be placed in a text file included with all distributions of the Covered
+Software under this License. Except to the extent prohibited by statute
+or regulation, such description must be sufficiently detailed for a
+recipient of ordinary skill to be able to understand it.
+
+5. Termination
+--------------
+
+5.1. The rights granted under this License will terminate automatically
+if You fail to comply with any of its terms. However, if You become
+compliant, then the rights granted under this License from a particular
+Contributor are reinstated (a) provisionally, unless and until such
+Contributor explicitly and finally terminates Your grants, and (b) on an
+ongoing basis, if such Contributor fails to notify You of the
+non-compliance by some reasonable means prior to 60 days after You have
+come back into compliance. Moreover, Your grants from a particular
+Contributor are reinstated on an ongoing basis if such Contributor
+notifies You of the non-compliance by some reasonable means, this is the
+first time You have received notice of non-compliance with this License
+from such Contributor, and You become compliant prior to 30 days after
+Your receipt of the notice.
+
+5.2. If You initiate litigation against any entity by asserting a patent
+infringement claim (excluding declaratory judgment actions,
+counter-claims, and cross-claims) alleging that a Contributor Version
+directly or indirectly infringes any patent, then the rights granted to
+You by any and all Contributors for the Covered Software under Section
+2.1 of this License shall terminate.
+
+5.3. In the event of termination under Sections 5.1 or 5.2 above, all
+end user license agreements (excluding distributors and resellers) which
+have been validly granted by You or Your distributors under this License
+prior to termination shall survive termination.
+
+************************************************************************
+*                                                                      *
+*  6. Disclaimer of Warranty                                           *
+*  -------------------------                                           *
+*                                                                      *
+*  Covered Software is provided under this License on an "as is"       *
+*  basis, without warranty of any kind, either expressed, implied, or  *
+*  statutory, including, without limitation, warranties that the       *
+*  Covered Software is free of defects, merchantable, fit for a        *
+*  particular purpose or non-infringing. The entire risk as to the     *
+*  quality and performance of the Covered Software is with You.        *
+*  Should any Covered Software prove defective in any respect, You     *
+*  (not any Contributor) assume the cost of any necessary servicing,   *
+*  repair, or correction. This disclaimer of warranty constitutes an   *
+*  essential part of this License. No use of any Covered Software is   *
+*  authorized under this License except under this disclaimer.         *
+*                                                                      *
+************************************************************************
+
+************************************************************************
+*                                                                      *
+*  7. Limitation of Liability                                          *
+*  --------------------------                                          *
+*                                                                      *
+*  Under no circumstances and under no legal theory, whether tort      *
+*  (including negligence), contract, or otherwise, shall any           *
+*  Contributor, or anyone who distributes Covered Software as          *
+*  permitted above, be liable to You for any direct, indirect,         *
+*  special, incidental, or consequential damages of any character      *
+*  including, without limitation, damages for lost profits, loss of    *
+*  goodwill, work stoppage, computer failure or malfunction, or any    *
+*  and all other commercial damages or losses, even if such party      *
+*  shall have been informed of the possibility of such damages. This   *
+*  limitation of liability shall not apply to liability for death or   *
+*  personal injury resulting from such party's negligence to the       *
+*  extent applicable law prohibits such limitation. Some               *
+*  jurisdictions do not allow the exclusion or limitation of           *
+*  incidental or consequential damages, so this exclusion and          *
+*  limitation may not apply to You.                                    *
+*                                                                      *
+************************************************************************
+
+8. Litigation
+-------------
+
+Any litigation relating to this License may be brought only in the
+courts of a jurisdiction where the defendant maintains its principal
+place of business and such litigation shall be governed by laws of that
+jurisdiction, without reference to its conflict-of-law provisions.
+Nothing in this Section shall prevent a party's ability to bring
+cross-claims or counter-claims.
+
+9. Miscellaneous
+----------------
+
+This License represents the complete agreement concerning the subject
+matter hereof. If any provision of this License is held to be
+unenforceable, such provision shall be reformed only to the extent
+necessary to make it enforceable. Any law or regulation which provides
+that the language of a contract shall be construed against the drafter
+shall not be used to construe this License against a Contributor.
+
+10. Versions of the License
+---------------------------
+
+10.1. New Versions
+
+Mozilla Foundation is the license steward. Except as provided in Section
+10.3, no one other than the license steward has the right to modify or
+publish new versions of this License. Each version will be given a
+distinguishing version number.
+
+10.2. Effect of New Versions
+
+You may distribute the Covered Software under the terms of the version
+of the License under which You originally received the Covered Software,
+or under the terms of any subsequent version published by the license
+steward.
+
+10.3. Modified Versions
+
+If you create software not governed by this License, and you want to
+create a new license for such software, you may create and use a
+modified version of this License if you rename the license and remove
+any references to the name of the license steward (except to note that
+such modified license differs from this License).
+
+10.4. Distributing Source Code Form that is Incompatible With Secondary
+Licenses
+
+If You choose to distribute Source Code Form that is Incompatible With
+Secondary Licenses under the terms of this version of the License, the
+notice described in Exhibit B of this License must be attached.
+
+Exhibit A - Source Code Form License Notice
+-------------------------------------------
+
+  This Source Code Form is subject to the terms of the Mozilla Public
+  License, v. 2.0. If a copy of the MPL was not distributed with this
+  file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+If it is not possible or desirable to put the notice in a particular
+file, then You may include the notice in a location (such as a LICENSE
+file in a relevant directory) where a recipient would be likely to look
+for such a notice.
+
+You may add additional accurate notices of copyright ownership.
+
+Exhibit B - "Incompatible With Secondary Licenses" Notice
+---------------------------------------------------------
+
+  This Source Code Form is "Incompatible With Secondary Licenses", as
+  defined by the Mozilla Public License, v. 2.0.

+ 168 - 0
README.md

@@ -0,0 +1,168 @@
+EliasDB
+=======
+EliasDB is a graph-based database which aims to provide a lightweight solution for projects which want to store their data as a graph. EliasDB does not require any third-party libraries.
+
+<p>
+<a href="https://devt.de/build_status.html"><img src="https://devt.de/nightly/build.eliasdb.svg" alt="Build status"></a>
+<a href="https://devt.de/nightly/test.eliasdb.html"><img src="https://devt.de/nightly/test.eliasdb.svg" alt="Code coverage"></a>
+<a href="https://goreportcard.com/report/github.com/krotik/eliasdb">
+<img src="https://goreportcard.com/badge/github.com/krotik/eliasdb?style=flat-square" alt="Go Report Card"></a>
+<a href="http://devt.de/docs/pkg/devt.de/eliasdb/">
+<img src="https://devt.de/nightly/godoc_badge.svg" alt="Go Doc"></a>
+<a href="https://gitter.im/eliasdb/Lobby">
+<img src="https://badges.gitter.im/gitterHQ/gitter.svg" alt="Gitter Chat"></a>
+</p>
+
+Features
+--------
+- Build on top of a fast key-value store which supports transactions and memory-only storage.
+- Data is stored in nodes (key-value objects) which are connected via edges.
+- Stored graphs can be separated via partitions.
+- Stored graphs support cascading deletions - delete one node and all its "children".
+- All stored data is indexed and can be quickly searched via a full text phrase search.
+- For more complex queries EliasDB has an own query language called EQL with an sql-like syntax.
+- Written in Go from scratch. No third party libraries were used apart from Go's standard library.
+- The database can be embedded or used as a standalone application.
+- When used as a standalone application it comes with an internal HTTPS webserver which
+  provides a REST API and a basic file server.
+- When used as an embedded database it supports transactions with rollbacks, iteration of data
+  and rule based consistency management.
+
+Getting Started (standalone application)
+----------------------------------------
+You can download a precompiled package for Windows (win64) or Linux (amd64) [here](https://devt.de/build_status.html).
+
+Extract it and execute the executable. The executable should automatically create 3 subfolders and a configuration file. It should start an HTTPS server on port 9090. To see a terminal point your webbrowser to:
+```
+https://localhost:9090/db/term.html
+```
+After accepting the self-signed certificate from the server you should see a web terminal. EliasDB can be stopped with a simple CTRL+C or by overwriting the content in eliasdb.lck with a single character.
+
+### Tutorial:
+
+To get an idea of what EliasDB is about have a look at the [tutorial](/examples/tutorial/doc/tutorial.md).
+
+### REST API:
+
+The terminal uses a REST API to communicate with the backend. The REST API can be browsed using a dynamically generated swagger.json definition (https://localhost:9090/db/swagger.json). You can browse the API of EliasDB's latest version [here](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/krotik/eliasdb/master/doc/swagger.json#/default).
+
+### Command line options
+The main EliasDB executable has two main tools:
+```
+Usage of ./eliasdb <tool>
+
+EliasDB graph based database
+
+Available commands:
+
+    console   EliasDB server console
+    server    Start EliasDB server
+```
+The most important one is server which starts the database server. The server has several options:
+```
+Usage of ./eliasdb server [options]
+
+  -export string
+    	Export the current database to a zip file
+  -help
+    	Show this help message
+  -import string
+    	Import a database from a zip file
+  -no-serv
+    	Do not start the server after initialization
+```
+Once the server is started the console tool can be used to interact with the server. The options of the console tool are:
+```
+Usage of ./eliasdb console [options]
+
+  -exec string
+    	Execute a single line and exit
+  -file string
+    	Read commands from a file and exit
+  -help
+    	Show this help message
+  -host string
+    	Host of the EliasDB server (default "localhost")
+  -port string
+    	Port of the EliasDB server (default "9090")
+```
+On the console type 'q' to exit and 'help' to get an overview of available commands:
+```
+Command Description
+export  Exports the last output.
+find    Do a full-text search of the database.
+help    Display descriptions for all available commands.
+info    Returns general database information.
+part    Displays or sets the current partition.
+ver     Displays server version information.
+```
+It is also possible to directly run EQL queries on the console. Use the arrow keys to cycle through the command history.
+
+### Configuration
+EliasDB uses a single configuration file called eliasdb.config.json. After starting EliasDB for the first time it should create a default configuration file. Available configurations are:
+
+| Configuration Option | Description |
+| --- | --- |
+| ClusterConfigFile | Cluster configuration file. |
+| ClusterLogHistory | File which is used to store the console history. |
+| ClusterStateInfoFile | File which is used to store the cluster state. |
+| CookieMaxAgeSeconds | Lifetime for cookies used by EliasDB. |
+| EnableAccessControl | Flag if access control for EliasDB should be enabled. This provides user authentication and authorization features. |
+| EnableCluster | Flag if EliasDB clustering support should be enabled. EXPERIMENTAL! |
+| EnableClusterTerminal | Flag if the cluster terminal file /web/db/cluster.html should be created. |
+| EnableReadOnly | Flag if the datastore should be open read-only. |
+| EnableWebFolder | Flag if the files in the webfolder /web should be served up by the webserver. If false only the REST API is accessible. |
+| EnableWebTerminal | Flag if the web terminal file /web/db/term.html should be created. |
+| HTTPSCertificate | Name of the webserver certificate which should be used. A new one is created if it does not exist. |
+| HTTPSHost | Hostname the webserver should listen to. This host is also used in the dynamically generated swagger definition. |
+| HTTPSKey | Name of the webserver private key which should be used. A new one is created if it does not exist. |
+| HTTPSPort | Port on which the webserver should listen on. |
+| LocationAccessDB | File which is used to store access control information. This file can be edited while the server is running and changes will be picked up immediately. |
+| LocationDatastore | Directory for datastore files. |
+| LocationHTTPS | Directory for the webserver's SSL related files. |
+| LocationUserDB | File which is used to store (hashed) user passwords. |
+| LocationWebFolder | Directory of the webserver's webfolder. |
+| LockFile | Lockfile for the webserver which will be watched duing runtime. Replacing the content of this file with a single character will shutdown the webserver gracefully. |
+| MemoryOnlyStorage | Flag if the datastore should only be kept in memory. |
+| ResultCacheMaxAgeSeconds | EQL queries create result sets which are cached. The value describes the amount of time in seconds a result is kept in the cache. |
+| ResultCacheMaxSize | EQL queries create result sets which are cached. The value describes the number of results which can be kept in the cache. |
+
+Note: It is not (and will never be) possible to access the REST API via HTTP.
+
+Building EliasDB
+----------------
+To build EliasDB from source you need to have Go installed. There a are two options:
+
+### Checkout from github (use this method if you want code + documentation and tutorials):
+
+Create a directory, change into it and run:
+```
+git clone https://github.com/krotik/eliasdb/ .
+```
+
+Assuming your GOPATH is set to the new directory you should be able to build the binary with:
+```
+go install devt.de/eliasdb/cli
+```
+
+### Using go get (use this method if you want to embed EliasDB in your project):
+
+Create a directory, change into it and run:
+```
+go get devt.de/common/... devt.de/eliasdb/...
+```
+
+Assuming your GOPATH is set to the new directory you should be able to build the binary with:
+```
+go build devt.de/eliasdb/cli
+```
+
+Further Reading
+---------------
+- A design document which describes the different components of the graph database. [Link](/doc/elias_db_design.md)
+- A reference for the EliasDB query language EQL. [Link](/doc/eql.md)
+- A quick overview of what you can do when you embed EliasDB in your own Go project. [Link](/doc/embedding.md)
+
+License
+-------
+EliasDB source code is available under the [Mozilla Public License](/LICENSE).

File diff suppressed because it is too large
+ 81 - 0
doc/elias_db_design.md


+ 308 - 0
doc/embedding.md

@@ -0,0 +1,308 @@
+EliasDB Code Tutorial
+=====================
+The following text will give you an introduction to EliasDB's code structure and how to embed EliasDB in another Go project.
+
+Getting the source code
+-----------------------
+The easiest way to get the source code of EliasDB is to use go get. Assuming you have a normal go project with GOROOT pointing to its root.
+You can checkout the source code of EliasDB with:
+```
+go get -d devt.de/common devt.de/eliasdb
+```
+For the rest of this tutorial it is assumed that you have the following directory structure:
+
+| Path | Description |
+| --- | --- |
+| src/devt.de/common | Common code used by EliasDB |
+| src/devt.de/eliasdb/ | Root directory for EliasDB containing the main package for the standalone server |
+| src/devt.de/eliasdb/api | HTTP endpoints for EliasDB's REST API |
+| src/devt.de/eliasdb/eql | Parser and interpreter for EQL |
+| src/devt.de/eliasdb/graph | API to the graph storage |
+| src/devt.de/eliasdb/hash | H-Tree implementation for EliasDB's underlying key-value store |
+| src/devt.de/eliasdb/storage | Low level storage API |
+| src/dect.de/eliasdb/version | Version file |
+
+For this tutorial we create a demo file:
+
+src/devt.de/demo/demo.go
+
+
+Simple graph database setup
+---------------------------
+The first step is to create a graph storage which will store the data. The following code will
+create a disk storage in the db/ subdirectory (the false flag opens the store in read / write mode):
+```
+func main() {
+
+	// Create a graph storage
+
+	gs, err := graphstorage.NewDiskGraphStorage("db", false)
+	if err != nil {
+		log.Fatal(err)
+		return
+	}
+	defer gs.Close()
+...
+```
+It is important to close a disk storage before shutdown. It is also possible to create a memory-only storage with:
+```
+	gs = graphstorage.NewMemoryGraphStorage("memdb")
+```
+
+After creating a storage we can now create a GraphManager object which provides the graph API:
+```
+	gm := graph.NewGraphManager(gs)
+
+```
+
+Storing and retrieving data
+---------------------------
+The main storage element in a graph database are nodes. All nodes stored in EliasDB are identified by a combination of key and kind. The node kind is basically the node type (e.g. Person) while the key is a node unique identifier.
+
+To store a single node in the datastore we can write the following code:
+```
+	node1 := data.NewGraphNode()
+	node1.SetAttr("key", "123")
+	node1.SetAttr("kind", "mynode")
+	node1.SetAttr("name", "Node1")
+	node1.SetAttr("text", "The first stored node")
+
+	gm.StoreNode("main", node1)
+```
+The attributes key and kind are compulsory. Storing a node with the same key and kind will overwrite any existing node. Each node should have a name which should be a human-readable label for the node. The StoreNode call gets a partition as the first argument. Nodes stored in separate partitions can not be linked by an edge. Search queries are scoped to a single partition.
+
+Nodes can be linked together via an edge:
+```
+	node2 := data.NewGraphNode()
+	node2.SetAttr(data.NodeKey, "456")
+	node2.SetAttr(data.NodeKind, "mynode")
+	node2.SetAttr(data.NodeName, "Node2")
+
+	gm.StoreNode("main", node2)
+
+	edge := data.NewGraphEdge()
+
+	edge.SetAttr(data.NodeKey, "abc")
+	edge.SetAttr(data.NodeKind, "myedge")
+
+	edge.SetAttr(data.EdgeEnd1Key, node1.Key())
+	edge.SetAttr(data.EdgeEnd1Kind, node1.Kind())
+	edge.SetAttr(data.EdgeEnd1Role, "node1")
+	edge.SetAttr(data.EdgeEnd1Cascading, true)
+
+	edge.SetAttr(data.EdgeEnd2Key, node2.Key())
+	edge.SetAttr(data.EdgeEnd2Kind, node2.Kind())
+	edge.SetAttr(data.EdgeEnd2Role, "node2")
+	edge.SetAttr(data.EdgeEnd2Cascading, false)
+
+	edge.SetAttr(data.NodeName, "Edge1")
+
+	gm.StoreEdge("main", edge)
+```
+Edges have more compulsory attributes than nodes. As well as key and kind for the edge itself, you also need to define for each end the key, kind, a role and a cascading flag. The cascading flag defines if delete actions to an end should be propagated to the other end. The role is a name which defines one end's relationship to the other. It is only used for traversals. An example relationship of nodes through an edge could be described like this:
+
+(Hans/Person) Father -- Family -- Child (Klaus/Person)
+
+We could traverse this relationship by writing:
+```
+    gm.Traverse("main", node1.Key(), node1.Kind(), "Father:Family:Child:Person", true)
+```
+The last boolean flag indicates if all data from the target node should be received. If set to false only the key and kind will be populated. If multiple edge kinds or roles should be traversed it is possible to use gm.TraverseMulti. Omitting a traversal component is like using a wildcard (e.g. :Family:: will traverse all family edges to any node kind).
+
+The storage of nodes and edges can be combined in a transaction. The transaction either inserts all items or none.
+```
+	trans := graph.NewGraphTrans(gm)
+	trans.StoreNode(...)
+	trans.StoreEdge(...)
+	trans.Commit()
+```
+Now that the datastore has some data we can use the graph API to query the data. To query a node you can use a lookup:
+```
+	n, err := gm.FetchNode("main", "123", "mynode")
+	fmt.Println(n, err)
+```
+To iterate over all nodes of a specific kind you can use a node iterator:
+```
+it, err := gm.NodeKeyIterator("main", "mynode")
+for it.HasNext() {
+	key := it.Next()
+	
+	if it.LastError != nil {
+		break
+	}
+
+	n, err := gm.FetchNode("main", key, "mynode")
+	fmt.Println(n, err)
+}
+```
+
+Querying the datastore
+----------------------
+Besides direct lookups and iterators the datastore also supports higher search functionality such as phrase searching and a query language.
+
+All data in the datastore is indexed. To query for a certain phrase you can run a phrase search:
+```
+idx, idxerr := gm.NodeIndexQuery("main", "mynode")
+if idxerr == nil {
+
+	keys, err := idx.LookupPhrase("text", "first stored")
+	if err == nil {
+
+		for _, key := range keys {
+			n, err := gm.FetchNode("main", key, "mynode")
+			fmt.Println(n, err)
+		}
+	}
+}
+```
+For even more complex searches you can use EQL (see also the EQL manual):
+```
+res, err := eql.RunQuery("myquery", "main", "get mynode where name = 'Node2'", gm)
+
+fmt.Println(res, err)
+```
+
+Adding REST API endpoints
+-------------------------
+EliasDB's REST API can be added easily when using Go's default webserver and router:
+```
+api.RegisterRestEndpoints(v1.V1EndpointMap)
+api.RegisterRestEndpoints(api.GeneralEndpointMap)
+```
+
+Example source
+--------------
+An example demo.go could look like this:
+```
+package demo
+
+import (
+	"fmt"
+	"log"
+
+	"devt.de/eliasdb/eql"
+	"devt.de/eliasdb/graph"
+	"devt.de/eliasdb/graph/data"
+	"devt.de/eliasdb/graph/graphstorage"
+)
+
+func main() {
+
+	// Create a graph storage
+
+	//gs, err := graphstorage.NewDiskGraphStorage("db", false)
+	//if err != nil {
+	//		log.Fatal(err)
+	//		return
+	//	}
+	//defer gs.Close()
+
+	// For memory only storage do:
+
+	gs := graphstorage.NewMemoryGraphStorage("memdb")
+
+	gm := graph.NewGraphManager(gs)
+
+	// Create transaction
+
+	trans := graph.NewGraphTrans(gm)
+
+	// Store node1
+
+	node1 := data.NewGraphNode()
+	node1.SetAttr("key", "123")
+	node1.SetAttr("kind", "mynode")
+	node1.SetAttr("name", "Node1")
+	node1.SetAttr("text", "The first stored node")
+
+	if err := trans.StoreNode("main", node1); err != nil {
+		log.Fatal(err)
+	}
+
+	// Store node 2
+
+	node2 := data.NewGraphNode()
+	node2.SetAttr(data.NodeKey, "456")
+	node2.SetAttr(data.NodeKind, "mynode")
+	node2.SetAttr(data.NodeName, "Node2")
+
+	if err := trans.StoreNode("main", node2); err != nil {
+		log.Fatal(err)
+	}
+
+	if err := trans.Commit(); err != nil {
+		log.Fatal(err)
+	}
+
+	trans = graph.NewGraphTrans(gm)
+
+	// Store edge between nodes
+
+	edge := data.NewGraphEdge()
+
+	edge.SetAttr(data.NodeKey, "abc")
+	edge.SetAttr(data.NodeKind, "myedge")
+
+	edge.SetAttr(data.EdgeEnd1Key, node1.Key())
+	edge.SetAttr(data.EdgeEnd1Kind, node1.Kind())
+	edge.SetAttr(data.EdgeEnd1Role, "node1")
+	edge.SetAttr(data.EdgeEnd1Cascading, true)
+
+	edge.SetAttr(data.EdgeEnd2Key, node2.Key())
+	edge.SetAttr(data.EdgeEnd2Kind, node2.Kind())
+	edge.SetAttr(data.EdgeEnd2Role, "node2")
+	edge.SetAttr(data.EdgeEnd2Cascading, false)
+
+	edge.SetAttr(data.NodeName, "Edge1")
+
+	if err := gm.StoreEdge("main", edge); err != nil {
+		log.Fatal(err)
+	}
+
+	// Commit transaction
+
+	if err := trans.Commit(); err != nil {
+		log.Fatal(err)
+	}
+
+	// Demo traversal:
+
+	nodes, edges, err := gm.TraverseMulti("main", "123", "mynode", ":::", false)
+	fmt.Println("out1:", nodes, edges, err)
+
+	// Demo key iterator:
+
+	it, err := gm.NodeKeyIterator("main", "mynode")
+	for it.HasNext() {
+		key := it.Next()
+
+		if it.LastError != nil {
+			break
+		}
+
+		n, err := gm.FetchNode("main", key, "mynode")
+		fmt.Println("out2:", n, err)
+	}
+
+	// Demo full text search
+
+	idx, idxerr := gm.NodeIndexQuery("main", "mynode")
+	if idxerr == nil {
+
+		keys, err := idx.LookupPhrase("text", "first stored")
+		if err == nil {
+
+			for _, key := range keys {
+				n, err := gm.FetchNode("main", key, "mynode")
+				fmt.Println("out3:", n, err)
+			}
+		}
+	}
+
+	// Demo eql query
+
+	res, err := eql.RunQuery("myquery", "main", "get mynode where name = 'Node2'", gm)
+
+	fmt.Println("out4:", res, err)
+}
+```

+ 130 - 0
doc/eql.md

@@ -0,0 +1,130 @@
+EliasDB query language
+======================
+
+EliasDB query language (EQL) is a query langugage to search nodes in a partition of the graph database. Its syntax is designed to follow natural language supporting complex graph queries while keeping simple queries simple. A simple EQL query has the following structure:
+```
+get <node kind> where <condition>
+```
+It reads: "Get all graph nodes of a certain node kind which match a certain condition". The condition is evaluated for each node from the specified kind. For example to get all "Person" nodes with the name "John" you could write:
+```
+get Person where name = John
+```
+The result of this query is a table listing all data store nodes which have a node attribute name with the value John.
+
+Where clause
+------------
+
+A where clause supports the following operators:
+
+- Standard boolean operators: and, or, not
+
+- Standard condition operators: =, !=, >, <, >=, <=, in, notin, contains, beginswith, endswith, containsnot
+
+- Standard arithmetic operators: +, -, *, /
+
+- Integer operations: // (integer division), % (modulo)
+
+- Regular expression operator: like
+
+Operators can be combined. Expressions can be segregated using parentheses. Each where condition should end in a boolean value. List operators such as “in” and “notin” operate on sequences of values which can be declared with square brackets e.g. [1,2,3].
+
+- Where clauses also support the following constants: true, false, null
+
+To explicitly define if a value represents a literal or a name of a node or edge attribute it is possible to prefix it with either 'attr:' for a node attribute name, 'eattr:' for an edge attribute name or 'val:' for a literal. In the majority of cases however the query interpreter will determine the right meaning. The precedence is: node attribute, edge attribute, literal value.
+
+EQL supports nested object structures on node attributes. A node value of { l1 : { l2 : { l3 : 123 } } } can be queried as:
+
+<attr name>.l1.l2.l3 = 123
+
+If the actual attribute name contins a dot then the 'attr:' prefix must be used.
+
+
+Traversal blocks
+----------------
+
+For the majority of useful queries it will be necessary to traverse the graph. Relationships between nodes can be matched with traversal specifications. A traversal specification has the following form:
+```
+<source role>:<relationship kind>:<destination role>:<destination kind>
+```
+All components of a traversal specification are optional. A traversal of all relationships can be expressed with:
+```
+:::
+```
+Traversal expressions in a query are defined as block expressions:
+```
+get <node kind> where <condition>
+ traverse <traversal spec> where <condition>
+    traverse <traversal spec> where <condition>
+        <Further traversals>
+    end
+end
+```
+Traversal expressions define which parts of the graph should be collected for the query. Reading from top to bottom each traversal expression defines a traversal step. Each traversal step will add several columns to the result if no explicit show clause is defined.
+
+Show clause
+-----------
+
+To control which data will be displayed in the final result it is possible to define a show clause. A show clause explicitly states which columns should be displayed in the result table.
+```
+get <node kind> where <condition>
+ traverse <traversal spec> where <condition>
+    traverse <traversal spec> where <condition>
+        <Further traversals>
+    end
+end
+show <show clause for column 1>, <show clause for column 2>, ...
+```
+The data can be defined by traversal position, as attribute name or as node/edge kind with attribute name.
+
+Examples:
+```
+1:n:key  - Display the key of the start nodes
+2:e:name - Display the name of the relationship from the 1 traversal step
+Person:name - Display the name of the first defined Person node from the query
+name – Display the name of the first defined node which has a name attribute
+```
+With clause
+-----------
+
+Operation which need to be applied once all rows of the result have been fetched can be defined in the with clause. If a with clause is defined it is always the last clause in a query.
+```
+get <node kind> show <show clauses> with <with operation>, <with operation>, ...
+```
+The following operations are possible:
+
+- ordering - Order a column (e.g. ordering(ascending Person:name) )
+             Available directives: ascending, descending
+ 
+- filtering - Filter a column (e.g. filtering(unique 2:e:name) )
+              Available directives: unique (column will only have unique values),
+                                    unique count (column will show unique values
+                                                  and a count of how many values were  
+                                                  encountered),
+                                    isnotnull (column will only contain not null 
+                                               values)
+- nulltraversal – Only includes rows in the result where all traversals steps
+                  where executed (i.e. do not include partial traversals)
+                  Available directives: true, false
+
+Functions
+---------
+
+Functions can be used to construct result values. A function can be used inside a where clause and inside a show clause. All function start with an “@” sign.
+
+Functions for conditions:
+```
+@count(<traversal spec>) - Counts how many nodes can be reached via a given spec from the traversal step of the condition.
+```
+
+```
+@parseDate(<date string>, <opt. layout>) - Converts a given date string into an unix time integer. The optional second parameter is the parsing layout stated as reference time (Mon Jan 2 15:04:05 -0700 MST 2006) - e.g. '2006-01-02' interprets <year>-<month>-<day> strings. The default layout is RFC3339.
+```
+
+Functions for the show clause:
+```
+@count(<traversal step>, <traversal spec>) - Counts how many nodes can be reached via a given spec from a given traversal step.
+```
+
+```
+@objget(<traversal step>, <attribute name>, <path to value>) - Extracts a value from a nested object structure.
+```

File diff suppressed because it is too large
+ 1 - 0
doc/swagger.json


File diff suppressed because it is too large
+ 78 - 0
examples/tutorial/doc/tutorial.md


BIN
examples/tutorial/doc/tutorial1.png


BIN
examples/tutorial/doc/tutorial2.png


BIN
examples/tutorial/doc/tutorial3.png


+ 11 - 0
examples/tutorial/start.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+cd "$(dirname "$0")"
+
+if ! [ -d "run" ]; then
+  mkdir -p run
+  cd run
+  ../../../eliasdb server -import ../tutorial_data.zip
+else
+  cd run
+  ../../../eliasdb server
+fi

BIN
examples/tutorial/tutorial_data.zip


+ 121 - 0
src/devt.de/common/bitutil/bitutil.go

@@ -0,0 +1,121 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+/*
+Package bitutil contains common function for bit-level operations.
+
+Pack and Unpack functions are used to pack and unpack a list of non-zero numbers
+very efficiently.
+*/
+package bitutil
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+)
+
+/*
+CompareByteArray compares the contents of two byte array slices. Returns true
+if both slices are equivalent in terms of size and content. The capacity may
+be different.
+*/
+func CompareByteArray(arr1 []byte, arr2 []byte) bool {
+	if len(arr1) != len(arr2) {
+		return false
+	}
+	for i, v := range arr1 {
+		if v != arr2[i] {
+			return false
+		}
+	}
+	return true
+}
+
+/*
+ByteSizeString takes a numeric byte size and returns it in human readable form.
+The useISU parameter determines which units to use. False uses the more common
+binary form. The units kibibyte, mebibyte, etc were established by the
+International Electrotechnical Commission (IEC) in 1998.
+
+useISU = True -> Decimal (as formally defined in the International System of Units)
+Bytes / Metric
+1000^1 kB kilobyte
+1000^2 MB megabyte
+1000^3 GB gigabyte
+1000^4 TB terabyte
+1000^5 PB petabyte
+1000^6 EB exabyte
+
+useISU = False -> Binary (as defined by the International Electrotechnical Commission)
+Bytes / Metric
+1024^1 KiB kibibyte
+1024^2 MiB mebibyte
+1024^3 GiB gibibyte
+1024^4 TiB tebibyte
+1024^5 PiB pebibyte
+1024^6 EiB exbibyte
+*/
+func ByteSizeString(size int64, useISU bool) string {
+	var byteSize, unit float64 = float64(size), 1024
+	var pre string
+
+	if useISU {
+		unit = 1000
+	}
+
+	if byteSize < unit {
+		return fmt.Sprintf("%d B", int(byteSize))
+	}
+
+	exp := math.Floor(math.Log(byteSize) / math.Log(unit))
+
+	if useISU {
+		pre = string("kMGTPE"[int(exp-1)])
+	} else {
+		pre = fmt.Sprintf("%vi", string("KMGTPE"[int(exp-1)]))
+	}
+
+	res := byteSize / math.Pow(unit, exp)
+
+	return fmt.Sprintf("%.1f %sB", res, pre)
+}
+
+/*
+HexDump produces a more-or-less human readable hex dump from a given byte array
+slice.
+*/
+func HexDump(data []byte) string {
+	buf := new(bytes.Buffer)
+	line := new(bytes.Buffer)
+
+	buf.WriteString("====\n000000  ")
+
+	for i, b := range data {
+
+		if i != 0 && i%10 == 0 {
+			buf.WriteString(fmt.Sprintf(" %s\n%06x  ", line.String(), i))
+			line = new(bytes.Buffer)
+		}
+
+		buf.WriteString(fmt.Sprintf("%02X ", b))
+		line.WriteString(fmt.Sprintf("%c", b))
+	}
+
+	rest := len(data) % 10
+	if rest != 0 {
+		for i := rest; i < 10; i++ {
+			buf.WriteString("   ")
+		}
+	}
+
+	buf.WriteString(fmt.Sprintf(" %s\n====\n", line.String()))
+
+	return buf.String()
+}

+ 81 - 0
src/devt.de/common/bitutil/bitutil_test.go

@@ -0,0 +1,81 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"testing"
+)
+
+func TestCompareByteArray(t *testing.T) {
+	testdata1 := []byte("Test")
+	testdata2 := make([]byte, 4, 5)
+	testdata3 := make([]byte, 3, 3)
+
+	if CompareByteArray(testdata1, testdata2) {
+		t.Error("Byte arrays should not be considered equal before copying data.")
+	}
+
+	if CompareByteArray(testdata1, testdata3) {
+		t.Error("Byte arrays should not be considered equal if the length is different.")
+	}
+
+	copy(testdata2, testdata1)
+
+	if cap(testdata1) == cap(testdata2) {
+		t.Error("Capacity of testdata sclices should be different.")
+	}
+
+	if !CompareByteArray(testdata1, testdata2) {
+		t.Error("Byte arrays should be considered equal.")
+	}
+}
+
+func TestByteSizeString(t *testing.T) {
+	// Test byte sizes
+	testdata := []int64{10000, 1024, 500, 1233456, 44166037, 84166037, 5000000000}
+
+	// non-ISU values
+	expected1 := []string{"9.8 KiB", "1.0 KiB", "500 B", "1.2 MiB", "42.1 MiB", "80.3 MiB", "4.7 GiB"}
+
+	// ISU values
+	expected2 := []string{"10.0 kB", "1.0 kB", "500 B", "1.2 MB", "44.2 MB", "84.2 MB", "5.0 GB"}
+
+	for i, test := range testdata {
+		res := ByteSizeString(test, false)
+		if res != expected1[i] {
+			t.Error("Unexpected value for non-isu value:", test,
+				"got:", res, "expected:", expected1[i])
+			return
+		}
+
+		res = ByteSizeString(test, true)
+		if res != expected2[i] {
+			t.Error("Unexpected value for isu value:", test,
+				"got:", res, "expected:", expected2[i])
+			return
+		}
+	}
+}
+
+func TestHexDump(t *testing.T) {
+	testdata := []byte("This is a test text. This is a test text.")
+
+	res := HexDump(testdata)
+	if res != "====\n"+
+		"000000  54 68 69 73 20 69 73 20 61 20  This is a \n"+
+		"00000a  74 65 73 74 20 74 65 78 74 2E  test text.\n"+
+		"000014  20 54 68 69 73 20 69 73 20 61   This is a\n"+
+		"00001e  20 74 65 73 74 20 74 65 78 74   test text\n"+
+		"000028  2E                             .\n"+
+		"====\n" {
+
+		t.Error("Invalid boundaries should cause an error")
+	}
+}

+ 89 - 0
src/devt.de/common/bitutil/murmurhash3.go

@@ -0,0 +1,89 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import "fmt"
+
+const (
+	c1 uint32 = 0xcc9e2d51
+	c2 uint32 = 0x1b873593
+)
+
+/*
+MurMurHashData hashes a given array of bytes. This is an implementation
+of Austin Appleby's MurmurHash3 (32bit) function.
+
+Reference implementation: http://code.google.com/p/smhasher/wiki/MurmurHash3
+*/
+func MurMurHashData(data []byte, offset int, size int, seed int) (uint32, error) {
+
+	// Check parameters
+
+	if offset < 0 || size < 0 {
+		return 0, fmt.Errorf("Invalid data boundaries; offset: %v; size: %v",
+			offset, size)
+	}
+
+	h1 := uint32(seed)
+	end := offset + size
+	end -= end % 4
+
+	// Check length of available data
+
+	if len(data) <= end {
+		return 0, fmt.Errorf("Data out of bounds; set boundary: %v; data length: %v",
+			end, len(data))
+	}
+
+	for i := offset; i < end; i += 4 {
+
+		var k1 = uint32(data[i])
+		k1 |= uint32(data[i+1]) << 8
+		k1 |= uint32(data[i+2]) << 16
+		k1 |= uint32(data[i+3]) << 24
+
+		k1 *= c1
+		k1 = (k1 << 15) | (k1 >> 17) // ROTL32(k1,15);
+		k1 *= c2
+
+		h1 ^= k1
+		h1 = (h1 << 13) | (h1 >> 19) // ROTL32(h1,13);
+		h1 = h1*5 + 0xe6546b64
+	}
+
+	// Tail
+
+	var k1 uint32
+
+	switch size & 3 {
+	case 3:
+		k1 = uint32(data[end+2]) << 16
+		fallthrough
+	case 2:
+		k1 |= uint32(data[end+1]) << 8
+		fallthrough
+	case 1:
+		k1 |= uint32(data[end])
+		k1 *= c1
+		k1 = (k1 << 15) | (k1 >> 17) // ROTL32(k1,15);
+		k1 *= c2
+		h1 ^= k1
+	}
+
+	h1 ^= uint32(size)
+
+	h1 ^= h1 >> 16
+	h1 *= 0x85ebca6b
+	h1 ^= h1 >> 13
+	h1 *= 0xc2b2ae35
+	h1 ^= h1 >> 16
+
+	return h1, nil
+}

+ 75 - 0
src/devt.de/common/bitutil/murmurhash3_test.go

@@ -0,0 +1,75 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"testing"
+)
+
+var testData = []byte("Now is the time for all good men to come to the aid of their country")
+
+var resultArray1 = []uint32{
+	0x249cb285, 0xcae32c45, 0x49cc6fdd, 0x3c89b814, 0xdc9778bb, 0x6db6607a,
+	0x736df8ad, 0xd367e257, 0x59b32232, 0x2496a9b4, 0x01d69f33, 0x08454378,
+	0x4ad4f630, 0x0ae1ca05, 0x042bdb5b, 0xbf3592e8, 0x0ed8b048, 0xb86958db,
+	0xa74ca5b6, 0xb7982271, 0x10a77c40, 0x8caba8ef, 0xe5085ab6, 0x8ee964b8,
+	0x170f0222, 0x42dec76d, 0xc4ebe4e5, 0x3d246566, 0x64f1133e, 0x8a0597dd,
+	0x5b13cdb8, 0x1c723636, 0xc8b60a2f, 0xb572fe46, 0xb801f177, 0x71d44c64,
+	0x755aeff1, 0x66ba2eeb, 0x5cfec249, 0x5b9d603f, 0x4e916049, 0x07622306,
+	0x57d4271f, 0x3fa8e56a, 0x4b4fe703, 0x995e958d, 0xdaf48fbb, 0xbe381e68,
+	0xd4af5452, 0x6b8e4cdc, 0x3c7bbc57, 0xd834a3e0, 0x78665c77, 0x5ab0d747,
+	0x4b34afb7, 0xbce90104, 0x25a31264, 0xa348c314, 0xab9fb213, 0x48f40ea9,
+	0xa232f18e, 0xda12f11a, 0x7dcdfcfb, 0x24381ba8, 0x1a15737d, 0x32b1ea01,
+	0x7ed7f6c6, 0xd16ab3ed}
+
+func TestMurMurHashData(t *testing.T) {
+
+	data := []byte{0xf6, 0x02, 0x03, 0x04}
+
+	// Test invalid data boundaries
+
+	_, err := MurMurHashData(data, 1, -3, 6)
+
+	if err == nil {
+		t.Error("Invalid boundaries should cause an error")
+	} else if err.Error() != "Invalid data boundaries; offset: 1; size: -3" {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	_, err = MurMurHashData(data, 1, 5, 6)
+
+	if err == nil {
+		t.Error("Invalid boundaries should cause an error")
+	} else if err.Error() != "Data out of bounds; set boundary: 4; data length: 4" {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	// Test against data
+
+	// Go source code is always UTF-8, so the string literal is UTF-8 text.
+	data = []byte("Now is the time for all good men to come to the aid of their country")
+
+	doTest := func(offset, size int) uint32 {
+		res, err := MurMurHashData(data, offset, size, 4)
+
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+
+		return res
+	}
+
+	for i := 0; i < len(resultArray1); i++ {
+		res := doTest(0, i)
+		if res != resultArray1[i] {
+			t.Errorf("Unexpected result; Expected: 0x%x; Got: 0x%x", resultArray1[i], res)
+		}
+	}
+}

+ 416 - 0
src/devt.de/common/bitutil/packedlist.go

@@ -0,0 +1,416 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"bytes"
+	"encoding/binary"
+	"math"
+)
+
+/*
+Different types of list packing
+*/
+const (
+	packListType2Bit = 0x1
+	packListType3Bit = 0x2
+	packListType6Bit = 0x3
+	packListTypeVar  = 0x0
+)
+
+/*
+PackList packs a given list to a string. Depending on the given highest number the
+list is packed in the most efficient way.
+*/
+func PackList(unpackedlist []uint64, highest uint64) string {
+
+	// Depending on the highest number convert to given list
+
+	switch {
+	case highest <= 3:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList2Bit(list)
+
+	case highest <= 7:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList3Bit(list)
+
+	case highest <= 63:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList6Bit(list)
+
+	case highest <= math.MaxUint8:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList8Bit(list)
+
+	case highest <= math.MaxUint16:
+		list := make([]uint16, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = uint16(num)
+		}
+		return PackList16Bit(list)
+
+	case highest <= math.MaxUint32:
+		list := make([]uint32, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = uint32(num)
+		}
+		return PackList32Bit(list)
+	}
+
+	return PackList64Bit(unpackedlist)
+}
+
+/*
+UnpackList unpacks a list from a packed string.
+*/
+func UnpackList(packedlist string) []uint64 {
+	plist := []byte(packedlist)
+
+	if len(plist) == 0 {
+		return nil
+	}
+
+	if plist[0]&0xC0 == packListTypeVar {
+		return UnpackBigList(packedlist)
+	}
+
+	res := UnpackSmallList(packedlist)
+	ret := make([]uint64, len(res))
+
+	for i, item := range res {
+		ret[i] = uint64(item)
+	}
+
+	return ret
+}
+
+/*
+PackList8Bit packs a list of 8 bit numbers.
+*/
+func PackList8Bit(list []uint8) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x00)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+PackList16Bit packs a list of 16 bit numbers.
+*/
+func PackList16Bit(list []uint16) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x01)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+PackList32Bit packs a list of 32 bit numbers.
+*/
+func PackList32Bit(list []uint32) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x02)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+PackList64Bit packs a list of 64 bit numbers.
+*/
+func PackList64Bit(list []uint64) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x03)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+UnpackBigList unpacks a list which has large values.
+*/
+func UnpackBigList(packedlist string) []uint64 {
+	var ret []uint64
+	plist := []byte(packedlist)
+
+	numlist := plist[1:]
+	reader := bytes.NewReader(numlist)
+
+	if plist[0] == 0x00 {
+		var item uint8
+		size := len(numlist)
+		ret = make([]uint64, size)
+		for i := 0; i < size; i++ {
+			binary.Read(reader, binary.LittleEndian, &item)
+			ret[i] = uint64(item)
+		}
+	} else if plist[0] == 0x01 {
+		var item uint16
+		size := len(numlist) / 2
+		ret = make([]uint64, size)
+		for i := 0; i < size; i++ {
+			binary.Read(reader, binary.LittleEndian, &item)
+			ret[i] = uint64(item)
+		}
+	} else if plist[0] == 0x02 {
+		var item uint32
+		size := len(numlist) / 4
+		ret = make([]uint64, size)
+		for i := 0; i < size; i++ {
+			binary.Read(reader, binary.LittleEndian, &item)
+			ret[i] = uint64(item)
+		}
+	} else if plist[0] == 0x03 {
+		size := len(numlist) / 8
+		ret = make([]uint64, size)
+		binary.Read(reader, binary.LittleEndian, ret)
+	}
+
+	return ret
+}
+
+/*
+PackList2Bit packs a list of bytes into a string using 2 bits for each item.
+(Items must be between 1 and 3)
+*/
+func PackList2Bit(list []byte) string {
+	if len(list) == 0 {
+		return ""
+	}
+
+	// Packing the list with 2 bit items reduces the size by a factor of 4
+
+	ret := make([]byte, int(math.Ceil(float64(1)/3+float64(len(list)-1)/4)))
+
+	if len(list) == 1 {
+		ret[0] = list2byte2bit(packListType2Bit, list[0], 0, 0)
+	} else if len(list) == 2 {
+		ret[0] = list2byte2bit(packListType2Bit, list[0], list[1], 0)
+	} else {
+		ret[0] = list2byte2bit(packListType2Bit, list[0], list[1], list[2])
+
+		j := 1
+		for i := 3; i < len(list); i += 4 {
+			if len(list[i:]) == 1 {
+				ret[j] = list2byte2bit(list[i], 0, 0, 0)
+			} else if len(list[i:]) == 2 {
+				ret[j] = list2byte2bit(list[i], list[i+1], 0, 0)
+			} else if len(list[i:]) == 3 {
+				ret[j] = list2byte2bit(list[i], list[i+1], list[i+2], 0)
+			} else {
+				ret[j] = list2byte2bit(list[i], list[i+1], list[i+2], list[i+3])
+			}
+			j++
+		}
+	}
+
+	return string(ret)
+}
+
+/*
+PackList3Bit packs a list of bytes into a string using 3 bits for each item.
+(Items must be between 1 and 7)
+*/
+func PackList3Bit(list []byte) string {
+	if len(list) == 0 {
+		return ""
+	}
+
+	// Packing the list with 2 bit items reduces the size by a factor of 2
+
+	ret := make([]byte, int(math.Ceil(float64(len(list))/2)))
+
+	if len(list) == 1 {
+		ret[0] = list2byte3bitAndHeader(packListType3Bit, list[0], 0)
+	} else {
+		ret[0] = list2byte3bitAndHeader(packListType3Bit, list[0], list[1])
+
+		j := 1
+		for i := 2; i < len(list); i += 2 {
+			if len(list[i:]) == 1 {
+				ret[j] = list2byte3bitAndHeader(0, list[i], 0)
+			} else {
+				ret[j] = list2byte3bitAndHeader(0, list[i], list[i+1])
+			}
+			j++
+		}
+	}
+
+	return string(ret)
+}
+
+/*
+PackList6Bit packs a list of bytes into a string using 6 bits for each item.
+(Items must be between 1 and 63)
+*/
+func PackList6Bit(list []byte) string {
+	if len(list) == 0 {
+		return ""
+	}
+
+	// Packing the list with 6 bit items does not reduce the factor
+
+	ret := make([]byte, len(list))
+
+	if len(list) == 1 {
+		ret[0] = list2byte6bitAndHeader(packListType6Bit, list[0])
+	} else {
+		ret[0] = list2byte6bitAndHeader(packListType6Bit, list[0])
+
+		for i := 1; i < len(list); i++ {
+			ret[i] = list2byte6bitAndHeader(0, list[i])
+		}
+	}
+
+	return string(ret)
+}
+
+/*
+UnpackSmallList unpacks a string into a list of bytes. Returns the list of bytes
+or a list of a single 0x00 byte if the numbers in the list are too big.
+*/
+func UnpackSmallList(packedlist string) []byte {
+	plist := []byte(packedlist)
+
+	if len(plist) == 0 {
+		return []byte{}
+	}
+
+	ltype := plist[0] & 0xC0 >> 6
+
+	if ltype == packListType2Bit {
+		return unpacklist2bit(plist)
+	} else if ltype == packListType3Bit {
+		return unpacklist3bit(plist)
+	} else if ltype == packListType6Bit {
+		return unpacklist6bit(plist)
+	}
+
+	// Must be gob encoded
+
+	return []byte{00}
+}
+
+func unpacklist2bit(packedlist []byte) []byte {
+	ret := make([]byte, 0, len(packedlist)*3)
+
+	for i := 0; i < len(packedlist); i++ {
+		b1, b2, b3, b4 := byte2list2bit(packedlist[i])
+		if i > 0 && b1 != 0 {
+			ret = append(ret, b1)
+		}
+		if b2 != 0 {
+			ret = append(ret, b2)
+		}
+		if b3 != 0 {
+			ret = append(ret, b3)
+		}
+		if b4 != 0 {
+			ret = append(ret, b4)
+		}
+	}
+
+	return ret
+}
+
+func unpacklist3bit(packedlist []byte) []byte {
+	ret := make([]byte, 0, len(packedlist)*2)
+
+	for i := 0; i < len(packedlist); i++ {
+		b1, b2 := byte2list3bit(packedlist[i])
+		if b1 != 0 {
+			ret = append(ret, b1)
+		}
+		if b2 != 0 {
+			ret = append(ret, b2)
+		}
+	}
+
+	return ret
+}
+
+func unpacklist6bit(packedlist []byte) []byte {
+	ret := make([]byte, 0, len(packedlist))
+
+	for i := 0; i < len(packedlist); i++ {
+		ret = append(ret, byte2list6bit(packedlist[i]))
+	}
+
+	return ret
+}
+
+func byte2list2bit(b byte) (b1 byte, b2 byte, b3 byte, b4 byte) {
+	b1 = b & 0xC0 >> 6
+	b2 = b & 0x30 >> 4
+	b3 = b & 0x0C >> 2
+	b4 = b & 0x03
+
+	return b1, b2, b3, b4
+}
+
+func list2byte2bit(b1 byte, b2 byte, b3 byte, b4 byte) byte {
+	return (b1 & 0x03 << 6) |
+		(b2 & 0x03 << 4) |
+		(b3 & 0x03 << 2) |
+		(b4 & 0x03)
+}
+
+func list2byte3bitAndHeader(b1 byte, b2 byte, b3 byte) byte {
+	return (b1 & 0x03 << 6) |
+		(b2 & 0x07 << 3) |
+		(b3 & 0x07)
+}
+
+func byte2list3bit(b byte) (b2 byte, b3 byte) {
+	b2 = b & 0x38 >> 3
+	b3 = b & 0x07
+
+	return b2, b3
+}
+
+func list2byte6bitAndHeader(b1 byte, b2 byte) byte {
+	return (b1 & 0x03 << 6) |
+		(b2 & 0x3F)
+}
+
+func byte2list6bit(b byte) byte {
+	return b & 0x3F
+}

+ 360 - 0
src/devt.de/common/bitutil/packedlist_test.go

@@ -0,0 +1,360 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"fmt"
+	"math"
+	"testing"
+)
+
+func TestListPacking(t *testing.T) {
+	mylist := make([]uint64, 7)
+	mylist[0] = 3
+	mylist[1] = 7
+	mylist[2] = 63
+	mylist[3] = math.MaxUint8
+	mylist[4] = math.MaxUint16
+	mylist[5] = math.MaxUint32
+	mylist[6] = math.MaxUint64
+
+	res := UnpackList(PackList(mylist, 3))
+	if res[0] != 3 {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res = UnpackList(PackList(mylist, 7))
+	if fmt.Sprint(res[:2]) != "[3 7]" {
+		t.Error("Unexpected result:", res[:2])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, 63))
+	if fmt.Sprint(res[:3]) != "[3 7 63]" {
+		t.Error("Unexpected result:", res[:3])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint8))
+	if fmt.Sprint(res[:4]) != "[3 7 63 255]" {
+		t.Error("Unexpected result:", res[:4])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint16))
+	if fmt.Sprint(res[:5]) != "[3 7 63 255 65535]" {
+		t.Error("Unexpected result:", res[:5])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint32))
+	if fmt.Sprint(res[:6]) != "[3 7 63 255 65535 4294967295]" {
+		t.Error("Unexpected result:", res[:6])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint64))
+	if fmt.Sprint(res[:7]) != "[3 7 63 255 65535 4294967295 18446744073709551615]" {
+		t.Error("Unexpected result:", res[:7])
+		return
+	}
+
+	res = UnpackList(PackList([]uint64{10, 12, 80}, 80))
+	if fmt.Sprint(res) != "[10 12 80]" {
+		t.Error("Unexpected result:", res)
+		return
+	}
+}
+
+func TestListPacking8(t *testing.T) {
+	list1 := PackList3Bit([]byte{1, 2, 3, 4, 5, 6, 7})
+	list2 := PackList16Bit([]uint16{1, 2, 3, 4})
+
+	if len(list1) != 4 || len(list2) != 9 {
+		t.Error("Unexpected lengths:", len(list1), len(list2))
+		return
+	}
+
+	res1 := UnpackList(list1)
+	res2 := UnpackList(list2)
+
+	if fmt.Sprint(res1) != "[1 2 3 4 5 6 7]" {
+		t.Error("Unexpected result:", res1)
+		return
+	}
+	if fmt.Sprint(res2) != "[1 2 3 4]" {
+		t.Error("Unexpected result:", res2)
+		return
+	}
+
+	if UnpackList("") != nil {
+		t.Error("Unexpected result")
+		return
+	}
+}
+
+func TestVarBitListPacking8(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint8, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint8
+	}
+
+	res := PackList8Bit(testlist)
+
+	if len(res) != scale+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint8(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestVarBitListPacking16(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint16, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint16
+	}
+
+	res := PackList16Bit(testlist)
+
+	if len(res) != scale*2+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint16(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestVarBitListPacking32(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint32, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint32
+	}
+
+	res := PackList32Bit(testlist)
+
+	if len(res) != scale*4+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint32(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestVarBitListPacking64(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint64, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint64
+	}
+
+	res := PackList64Bit(testlist)
+
+	if len(res) != scale*8+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint64(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestSmallListPacking(t *testing.T) {
+
+	// Test simple cases
+
+	if PackList2Bit([]byte{}) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if PackList3Bit([]byte{}) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if PackList6Bit([]byte{}) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if string(UnpackSmallList("")) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	// Simulates a gob encoded string
+
+	if string(UnpackSmallList(string([]byte{0x00}))) != string(0x00) {
+		t.Error("Unexpected result")
+		return
+	}
+
+	// Test normal cases
+
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 1, 2, 3}, []byte{0x5b, 0x6c}, 2)
+	checkListAndPresentation2bit(t, []byte{1}, []byte{0x50}, 1)
+	checkListAndPresentation2bit(t, []byte{1, 2}, []byte{0x58}, 1)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3}, []byte{0x5B}, 1)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 3}, []byte{0x5B, 0xC0}, 2)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 3, 2}, []byte{0x5B, 0xE0}, 2)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 3, 2, 1, 3}, []byte{0x5B, 0xE7}, 2)
+
+	checkListAndPresentation3bit(t, []byte{1, 2, 3, 1, 2, 3}, []byte{0x8A, 0x19, 0x13}, 3)
+	checkListAndPresentation3bit(t, []byte{1}, []byte{0x88}, 1)
+	checkListAndPresentation3bit(t, []byte{1, 2}, []byte{0x8A}, 1)
+	checkListAndPresentation3bit(t, []byte{1, 2, 3}, []byte{0x8A, 0x18}, 2)
+	checkListAndPresentation3bit(t, []byte{1, 2, 3, 3}, []byte{0x8A, 0x1B}, 2)
+	checkListAndPresentation3bit(t, []byte{1, 2, 3, 4, 5, 6, 7}, []byte{0x8A, 0x1C, 0x2E, 0x38}, 4)
+
+	checkListAndPresentation6bit(t, []byte{1, 2, 3, 1, 2, 3})
+	checkListAndPresentation6bit(t, []byte{1})
+	checkListAndPresentation6bit(t, []byte{1, 2})
+	checkListAndPresentation6bit(t, []byte{1, 2, 3})
+	checkListAndPresentation6bit(t, []byte{1, 2, 3, 3})
+	checkListAndPresentation6bit(t, []byte{1, 2, 3, 4, 35, 45, 63})
+}
+
+func checkListAndPresentation2bit(t *testing.T, list []byte, packedlist []byte, packedLen int) {
+	res := PackList2Bit(list)
+	if res != string(packedlist) {
+		t.Errorf("Unexpected result: %X", []byte(res))
+		return
+	}
+	if len(res) != packedLen {
+		t.Error("Unexpected size", len(res))
+		return
+	}
+	if dres := UnpackSmallList(res); string(dres) != string(list) {
+		t.Errorf("Unexpected result: %X", []byte(dres))
+		return
+	}
+}
+
+func checkListAndPresentation3bit(t *testing.T, list []byte, packedlist []byte, packedLen int) {
+	res := PackList3Bit(list)
+	if res != string(packedlist) {
+		t.Errorf("Unexpected result: %X", []byte(res))
+		return
+	}
+	if len(res) != packedLen {
+		t.Error("Unexpected size", len(res))
+		return
+	}
+	if dres := UnpackSmallList(res); string(dres) != string(list) {
+		t.Errorf("Unexpected result: %X", []byte(dres))
+		return
+	}
+}
+
+func checkListAndPresentation6bit(t *testing.T, list []byte) {
+	res := PackList6Bit(list)
+
+	packedlist := make([]byte, len(list))
+	copy(packedlist, list)
+	packedlist[0] = packedlist[0] | 0xC0
+
+	if res != string(packedlist) {
+		t.Errorf("Unexpected result: %X vs %X", []byte(res), packedlist)
+		return
+	}
+	if len(res) != len(list) {
+		t.Error("Unexpected size", len(res))
+		return
+	}
+	if dres := UnpackSmallList(res); string(dres) != string(list) {
+		t.Errorf("Unexpected result: %X", []byte(dres))
+		return
+	}
+}
+
+func TestList2byte2bit(t *testing.T) {
+	if res := list2byte2bit(0x01, 0x2, 0x03, 0x01); res != 0x6D {
+		t.Errorf("Unexpected result: %X", res)
+		return
+	}
+	if res := list2byte3bitAndHeader(0x00, 0x07, 0x03); res != 0x3B {
+		t.Errorf("Unexpected result: %X", res)
+		return
+	}
+}
+
+func TestByte2list2bit(t *testing.T) {
+	if a, b, c, d := byte2list2bit(0x30); a != 00 || b != 03 || c != 00 || d != 00 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x80); a != 02 || b != 00 || c != 00 || d != 00 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x01); a != 00 || b != 00 || c != 00 || d != 01 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x31); a != 00 || b != 03 || c != 00 || d != 01 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x05); a != 00 || b != 00 || c != 01 || d != 01 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+}
+
+func TestByte2list3bit(t *testing.T) {
+	if a, b := byte2list3bit(0x01); a != 00 || b != 01 {
+		t.Error("Unexpected result:", a, b)
+		return
+	}
+	if a, b := byte2list3bit(0x31); a != 06 || b != 01 {
+		t.Error("Unexpected result:", a, b)
+		return
+	}
+	if a, b := byte2list3bit(0x05); a != 00 || b != 05 {
+		t.Error("Unexpected result:", a, b)
+		return
+	}
+}

+ 188 - 0
src/devt.de/common/cryptutil/gencert.go

@@ -0,0 +1,188 @@
+/*
+Package cryptutil contains cryptographic utility functions.
+
+Certificate generation code based on:
+go source src/crypto/tls/generate_cert.go
+
+Copyright 2009 The Go Authors. All rights reserved.
+Use of this source code is governed by a BSD-style license.
+*/
+package cryptutil
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"strings"
+	"time"
+)
+
+/*
+GenCert generates certificate files in a given path.
+
+path       - Path to generate the certificate in.
+certFile   - Certificate file to generate.
+keyFile    - Key file to generate.
+host       - Comma-separated hostnames and IPs to generate a certificate for.
+validFrom  - Creation date formatted as Jan 1 15:04:05 2011. Default is empty string which means now.
+validFor   - Duration that certificate is valid for. Default is 365*24*time.Hour.
+isCA       - Flag whether this cert should be its own Certificate Authority.
+rsaBits    - Size of RSA key to generate. Ignored if ecdsa-curve is set. Default is 2048.
+ecdsaCurve - ECDSA curve to use to generate a key. Valid values are P224, P256, P384, P521 or empty string (not set).
+*/
+func GenCert(path string, certFile string, keyFile string, host string,
+	validFrom string, validFor time.Duration, isCA bool, rsaBits int, ecdsaCurve string) error {
+
+	var err error
+
+	// Check parameters
+
+	if path != "" && !strings.HasSuffix(path, "/") {
+		path += "/"
+	}
+
+	if host == "" {
+		return errors.New("Host required for certificate generation")
+	}
+
+	var notBefore time.Time
+
+	if validFrom == "" {
+		notBefore = time.Now()
+	} else {
+		notBefore, err = time.Parse("Jan 2 15:04:05 2006", validFrom)
+		if err != nil {
+			return fmt.Errorf("Failed to parse creation date: %s", err)
+		}
+	}
+
+	notAfter := notBefore.Add(validFor)
+
+	// Generate private key
+
+	var priv interface{}
+
+	switch ecdsaCurve {
+	case "":
+		priv, err = rsa.GenerateKey(rand.Reader, rsaBits)
+	case "P224":
+		priv, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
+	case "P256":
+		priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	case "P384":
+		priv, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
+	case "P521":
+		priv, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
+	default:
+		err = fmt.Errorf("Unrecognized elliptic curve: %q", ecdsaCurve)
+	}
+
+	if err != nil {
+		return fmt.Errorf("Failed to generate private key: %s", err)
+	}
+
+	// Generate serial random number
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, _ := rand.Int(rand.Reader, serialNumberLimit)
+
+	// Create and populate the certificate template
+
+	template := x509.Certificate{
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			Organization: []string{"None"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	// Add hosts
+
+	hosts := strings.Split(host, ",")
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		} else {
+			template.DNSNames = append(template.DNSNames, h)
+		}
+	}
+
+	// Set the CA flag
+
+	if isCA {
+		template.IsCA = isCA
+		template.KeyUsage |= x509.KeyUsageCertSign
+	}
+
+	// Create the certificate and write it out
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
+
+	if err == nil {
+
+		certOut, err := os.Create(path + certFile)
+		defer certOut.Close()
+
+		if err != nil {
+			return fmt.Errorf("Failed to open %s for writing: %s", certFile, err)
+		}
+
+		pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+
+		// Write out private key
+
+		keyOut, err := os.OpenFile(path+keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+		defer keyOut.Close()
+
+		if err != nil {
+			return fmt.Errorf("Failed to open %v for writing: %v", keyFile, err)
+		}
+
+		pem.Encode(keyOut, pemBlockForKey(priv))
+	}
+
+	return err
+}
+
+/*
+Return public key from a given key pair.
+*/
+func publicKey(priv interface{}) interface{} {
+	switch k := priv.(type) {
+	case *rsa.PrivateKey:
+		return &k.PublicKey
+	case *ecdsa.PrivateKey:
+		return &k.PublicKey
+	default:
+		return nil
+	}
+}
+
+/*
+Return private key pem block for a given key pair.
+*/
+func pemBlockForKey(priv interface{}) *pem.Block {
+	switch k := priv.(type) {
+	case *rsa.PrivateKey:
+		return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
+	case *ecdsa.PrivateKey:
+		b, _ := x509.MarshalECPrivateKey(k)
+		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
+	default:
+		return nil
+	}
+}

+ 149 - 0
src/devt.de/common/cryptutil/gencert_test.go

@@ -0,0 +1,149 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"devt.de/common/fileutil"
+)
+
+const certDir = "certs"
+
+const invalidFileName = "**" + string(0x0)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// Setup
+	if res, _ := fileutil.PathExists(certDir); res {
+		os.RemoveAll(certDir)
+	}
+
+	err := os.Mkdir(certDir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Run the tests
+	res := m.Run()
+
+	// Teardown
+	err = os.RemoveAll(certDir)
+	if err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+
+	os.Exit(res)
+}
+
+func TestGenCert(t *testing.T) {
+
+	checkGeneration := func(ecdsaCurve string) error {
+
+		// Generate a certificate and private key
+
+		err := GenCert(certDir, "cert.pem", "key.pem", "localhost,127.0.0.1", "", 365*24*time.Hour, true, 2048, ecdsaCurve)
+		if err != nil {
+			return err
+		}
+
+		// Check that the files were generated
+
+		if ok, _ := fileutil.PathExists(certDir + "/key.pem"); !ok {
+			return errors.New("Private key was not generated")
+		}
+
+		if ok, _ := fileutil.PathExists(certDir + "/cert.pem"); !ok {
+			return errors.New("Certificate was not generated")
+		}
+
+		_, err = ReadX509CertsFromFile(certDir + "/cert.pem")
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if err := checkGeneration(""); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P224"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P256"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P384"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P521"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Test error cases
+
+	err := GenCert(certDir, "cert.pem", "key.pem", "", "", 365*24*time.Hour, true, 2048, "")
+	if err.Error() != "Host required for certificate generation" {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, "cert.pem", "key.pem", "localhost", "", 365*24*time.Hour, true, 2048, "xxx")
+	if err.Error() != `Failed to generate private key: Unrecognized elliptic curve: "xxx"` {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, "cert.pem", "key.pem", "localhost", "xxx", 365*24*time.Hour, true, 2048, "")
+	if err.Error() != `Failed to parse creation date: parsing time "xxx" as "Jan 2 15:04:05 2006": cannot parse "xxx" as "Jan"` {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, "cert.pem", invalidFileName, "localhost", "", 365*24*time.Hour, true, 2048, "")
+	if !strings.HasPrefix(err.Error(), "Failed to open") {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, invalidFileName, "key.pem", "localhost", "", 365*24*time.Hour, true, 2048, "")
+	if !strings.HasPrefix(err.Error(), "Failed to open") {
+		t.Error(err)
+		return
+	}
+
+	if publicKey(nil) != nil {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if pemBlockForKey(nil) != nil {
+		t.Error("Unexpected result")
+		return
+	}
+}

+ 109 - 0
src/devt.de/common/cryptutil/stringcrypt.go

@@ -0,0 +1,109 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"io"
+)
+
+/*
+EncryptString encrypts a given string using AES (cfb mode).
+*/
+func EncryptString(passphrase, text string) (string, error) {
+	var ret []byte
+
+	// Create a new cipher with the given key
+
+	key := sha256.Sum256([]byte(passphrase))
+
+	block, err := aes.NewCipher((&key)[:])
+
+	if err == nil {
+
+		// Base64 encode the string
+
+		b := base64.StdEncoding.EncodeToString([]byte(text))
+
+		ciphertext := make([]byte, aes.BlockSize+len(b))
+
+		// Create the initialization vector using random numbers
+
+		iv := ciphertext[:aes.BlockSize]
+
+		if _, err = io.ReadFull(rand.Reader, iv); err == nil {
+
+			// Do the encryption
+
+			cfb := cipher.NewCFBEncrypter(block, iv)
+
+			cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
+
+			ret = ciphertext
+		}
+	}
+
+	return string(ret), err
+}
+
+/*
+DecryptString decrypts a given string using AES (cfb mode).
+*/
+func DecryptString(passphrase, text string) (string, error) {
+	var ret []byte
+
+	// Check encrypted text
+
+	if len(text) < aes.BlockSize {
+		return "", fmt.Errorf("Ciphertext is too short - must be at least: %v", aes.BlockSize)
+	}
+
+	// Create a new cipher with the given key
+
+	key := sha256.Sum256([]byte(passphrase))
+
+	block, err := aes.NewCipher((&key)[:])
+
+	if err == nil {
+
+		// Separate initialization vector and actual encrypted text
+
+		iv := text[:aes.BlockSize]
+
+		text = text[aes.BlockSize:]
+
+		// Do the decryption
+
+		cfb := cipher.NewCFBDecrypter(block, []byte(iv))
+
+		ret = []byte(text) // Reuse text buffer
+
+		cfb.XORKeyStream(ret, []byte(text))
+
+		// Decode text from base64
+
+		ret, err = base64.StdEncoding.DecodeString(string(ret))
+
+		if err != nil {
+
+			// Return a proper error if something went wrong
+
+			ret = nil
+			err = fmt.Errorf("Could not decrypt data")
+		}
+	}
+
+	return string(ret), err
+}

+ 53 - 0
src/devt.de/common/cryptutil/stringcrypt_test.go

@@ -0,0 +1,53 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"testing"
+)
+
+func TestStringEncryption(t *testing.T) {
+
+	secret := "This is a test"
+
+	encString, err := EncryptString("foo", secret)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	decString, err := DecryptString("foo", encString)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if decString != secret {
+		t.Error("Unexpected result:", decString, secret)
+		return
+	}
+
+	decString, err = DecryptString("foo1", encString)
+	if err.Error() != "Could not decrypt data" {
+		t.Error(err)
+		return
+	}
+
+	if decString != "" {
+		t.Error("Unexpected result:", decString)
+		return
+	}
+
+	decString, err = DecryptString("foo1", "bar")
+	if err.Error() != "Ciphertext is too short - must be at least: 16" {
+		t.Error(err)
+		return
+	}
+}

+ 37 - 0
src/devt.de/common/cryptutil/uuid.go

@@ -0,0 +1,37 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"crypto/rand"
+
+	"devt.de/common/errorutil"
+)
+
+/*
+GenerateUUID generates a version 4 (randomly generated) UUID according to RFC4122.
+*/
+func GenerateUUID() [16]byte {
+	var u [16]byte
+
+	_, err := rand.Read(u[:])
+	errorutil.AssertOk(err)
+
+	// Set version 4
+
+	u[6] = (u[6] & 0x0f) | 0x40
+
+	// Set variant bits - variant of RFC 4122
+
+	u[8] = (u[8] & 0xbf) | 0x80
+
+	return u
+
+}

+ 22 - 0
src/devt.de/common/cryptutil/uuid_test.go

@@ -0,0 +1,22 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestUUID(t *testing.T) {
+	if fmt.Sprint(GenerateUUID()) == "" {
+		t.Error("Unexpected result")
+		return
+	}
+}

+ 109 - 0
src/devt.de/common/cryptutil/x509util.go

@@ -0,0 +1,109 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+/*
+ReadX509CertsFromFile reads a list of pem encoded certificates from a given file.
+*/
+func ReadX509CertsFromFile(filename string) ([]*x509.Certificate, error) {
+	var err error
+	var certs []*x509.Certificate
+
+	file, err := os.OpenFile(filename, os.O_RDONLY, 0660)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	certsString, err := ioutil.ReadAll(file)
+	if err == nil {
+		certs, err = ReadX509Certs(certsString)
+	}
+
+	return certs, err
+}
+
+/*
+ReadX509Certs reads a list of pem encoded certificates from a byte array.
+*/
+func ReadX509Certs(certs []byte) ([]*x509.Certificate, error) {
+
+	var blocks []byte
+
+	for {
+		var block *pem.Block
+
+		block, certs = pem.Decode(certs)
+		if block == nil {
+			return nil, errors.New("PEM not parsed")
+		}
+
+		blocks = append(blocks, block.Bytes...)
+		if len(certs) == 0 {
+			break
+		}
+	}
+	c, err := x509.ParseCertificates(blocks)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+/*
+Sha1CertFingerprint computes a sha1 fingerprint for a certificate.
+*/
+func Sha1CertFingerprint(cert *x509.Certificate) string {
+	return formatFingerprint(fmt.Sprintf("%x", sha1.Sum(cert.Raw)))
+}
+
+/*
+Sha256CertFingerprint computes a sha256 fingerprint for a certificate.
+*/
+func Sha256CertFingerprint(cert *x509.Certificate) string {
+	return formatFingerprint(fmt.Sprintf("%x", sha256.Sum256(cert.Raw)))
+}
+
+/*
+Md5CertFingerprint computes a md5 fingerprint for a certificate.
+*/
+func Md5CertFingerprint(cert *x509.Certificate) string {
+	return formatFingerprint(fmt.Sprintf("%x", md5.Sum(cert.Raw)))
+}
+
+/*
+Format a given fingerprint string.
+*/
+func formatFingerprint(raw string) string {
+	var buf bytes.Buffer
+
+	for i, c := range raw {
+		buf.WriteByte(byte(c))
+		if (i+1)%2 == 0 && i != len(raw)-1 {
+			buf.WriteByte(byte(':'))
+		}
+	}
+
+	return buf.String()
+}

+ 94 - 0
src/devt.de/common/cryptutil/x509util_test.go