Browse Source

feat: Adding various common code fragments

Matthias Ladkau 10 months ago
parent
commit
80031ffe1d
100 changed files with 21971 additions and 2 deletions
  1. 23 2
      README.md
  2. 121 0
      bitutil/bitutil.go
  3. 81 0
      bitutil/bitutil_test.go
  4. 89 0
      bitutil/murmurhash3.go
  5. 75 0
      bitutil/murmurhash3_test.go
  6. 416 0
      bitutil/packedlist.go
  7. 360 0
      bitutil/packedlist_test.go
  8. 189 0
      cryptutil/gencert.go
  9. 149 0
      cryptutil/gencert_test.go
  10. 109 0
      cryptutil/stringcrypt.go
  11. 53 0
      cryptutil/stringcrypt_test.go
  12. 37 0
      cryptutil/uuid.go
  13. 22 0
      cryptutil/uuid_test.go
  14. 109 0
      cryptutil/x509util.go
  15. 94 0
      cryptutil/x509util_test.go
  16. 65 0
      datautil/datacopy.go
  17. 80 0
      datautil/datacopy_test.go
  18. 246 0
      datautil/mapcache.go
  19. 148 0
      datautil/mapcache_test.go
  20. 52 0
      datautil/nesting.go
  21. 79 0
      datautil/nesting_test.go
  22. 102 0
      datautil/nonce.go
  23. 55 0
      datautil/nonce_test.go
  24. 112 0
      datautil/persistentmap.go
  25. 146 0
      datautil/persistentmap_test.go
  26. 181 0
      datautil/ringbuffer.go
  27. 115 0
      datautil/ringbuffer_test.go
  28. 701 0
      datautil/userdb.go
  29. 484 0
      datautil/userdb_test.go
  30. 30 0
      defs/rambazamba/eventsource.go
  31. 40 0
      defs/rumble/func.go
  32. 45 0
      defs/rumble/globals.go
  33. 22 0
      defs/rumble/runtime.go
  34. 27 0
      defs/rumble/variables.go
  35. 68 0
      errorutil/errorutil.go
  36. 64 0
      errorutil/errorutil_test.go
  37. 418 0
      fileutil/config.go
  38. 221 0
      fileutil/config_test.go
  39. 134 0
      fileutil/fileutil.go
  40. 135 0
      fileutil/fileutil_test.go
  41. 439 0
      fileutil/multifilebuffer.go
  42. 388 0
      fileutil/multifilebuffer_test.go
  43. 89 0
      fileutil/zip.go
  44. 79 0
      fileutil/zip_test.go
  45. 127 0
      flowutil/eventpump.go
  46. 238 0
      flowutil/eventpump_test.go
  47. 3 0
      go.mod
  48. 1284 0
      httputil/access/acl.go
  49. 1271 0
      httputil/access/acl_test.go
  50. 45 0
      httputil/auth/auth.go
  51. 194 0
      httputil/auth/auth_test.go
  52. 139 0
      httputil/auth/basic.go
  53. 222 0
      httputil/auth/basic_test.go
  54. 300 0
      httputil/auth/cookie.go
  55. 243 0
      httputil/auth/cookie_test.go
  56. 278 0
      httputil/httpserver.go
  57. 290 0
      httputil/httpserver_test.go
  58. 113 0
      httputil/user/session.go
  59. 151 0
      httputil/user/session_test.go
  60. 258 0
      httputil/user/user.go
  61. 186 0
      httputil/user/user_test.go
  62. 114 0
      httputil/util.go
  63. 144 0
      httputil/util_test.go
  64. 141 0
      imageutil/asciiraster.go
  65. 163 0
      imageutil/asciiraster_test.go
  66. 314 0
      imageutil/rasterfont1.go
  67. 852 0
      imageutil/rasterfont2.go
  68. 149 0
      lang/graphql/parser/const.go
  69. 487 0
      lang/graphql/parser/lexer.go
  70. 246 0
      lang/graphql/parser/lexer_test.go
  71. 126 0
      lang/graphql/parser/node.go
  72. 830 0
      lang/graphql/parser/parser.go
  73. 1236 0
      lang/graphql/parser/parser_test.go
  74. 66 0
      lang/graphql/parser/parsererrors.go
  75. 37 0
      lang/graphql/parser/runtime.go
  76. 227 0
      lockutil/lockfile.go
  77. 139 0
      lockutil/lockfile_test.go
  78. 111 0
      logutil/formatter.go
  79. 72 0
      logutil/formatter_test.go
  80. 299 0
      logutil/logger.go
  81. 179 0
      logutil/logger_test.go
  82. 34 0
      pools/pools.go
  83. 56 0
      pools/pools_test.go
  84. 514 0
      pools/threadpool.go
  85. 415 0
      pools/threadpool_test.go
  86. 83 0
      sortutil/heap.go
  87. 101 0
      sortutil/heap_test.go
  88. 227 0
      sortutil/priorityqueue.go
  89. 196 0
      sortutil/priorityqueue_test.go
  90. 62 0
      sortutil/sortutil.go
  91. 48 0
      sortutil/sortutil_test.go
  92. 118 0
      sortutil/vectorclock.go
  93. 97 0
      sortutil/vectorclock_test.go
  94. 710 0
      stringutil/stringutil.go
  95. 596 0
      stringutil/stringutil_test.go
  96. 163 0
      stringutil/transform.go
  97. 103 0
      stringutil/transform_test.go
  98. 246 0
      termutil/autoterm.go
  99. 236 0
      termutil/autoterm_test.go
  100. 0 0
      termutil/fileterm.go

+ 23 - 2
README.md

@@ -1,3 +1,24 @@
-# common
+Common
+--
+Common is a collection of common algorithms and functions which are used across a multitude of projects on devt.de.
 
-Common algorithms and functions
+|Package|Description|
+| --- | --- |
+| bitutil | Byte processing helper functions. |
+| cryptutil | Processing certificates, uuids and encryption/decryption helpers. |
+| datautil | Datastructures and data storage helpers. |
+| defs | Definition used across devt.de projects. |
+| errorutil | Helper functions around error processing. |
+| fileutil | File handling utilities. |
+| flowutil | Datastructures for flow control. |
+| httputil | Heplers for HTTP handling. |
+| imageutil | Image and pixel processing. |
+| lang | Lexer and parsers. |
+| lockutil | Utilities for locking. |
+| logutil | Simple logging infrastructure. |
+| pools | Pooling helpers. |
+| sortutil | Datastructures and utilities around sorting. |
+| stringutil | String processing helpers. |
+| termutil | Unix/Windows terminal helpers. |
+| testutil | Utilities for code testing. |
+| timeutil | Helpers for time processing. |

+ 121 - 0
bitutil/bitutil.go

@@ -0,0 +1,121 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+/*
+Package bitutil contains common function for bit-level operations.
+
+Pack and Unpack functions are used to pack and unpack a list of non-zero numbers
+very efficiently.
+*/
+package bitutil
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+)
+
+/*
+CompareByteArray compares the contents of two byte array slices. Returns true
+if both slices are equivalent in terms of size and content. The capacity may
+be different.
+*/
+func CompareByteArray(arr1 []byte, arr2 []byte) bool {
+	if len(arr1) != len(arr2) {
+		return false
+	}
+	for i, v := range arr1 {
+		if v != arr2[i] {
+			return false
+		}
+	}
+	return true
+}
+
+/*
+ByteSizeString takes a numeric byte size and returns it in human readable form.
+The useISU parameter determines which units to use. False uses the more common
+binary form. The units kibibyte, mebibyte, etc were established by the
+International Electrotechnical Commission (IEC) in 1998.
+
+useISU = True -> Decimal (as formally defined in the International System of Units)
+Bytes / Metric
+1000^1 kB kilobyte
+1000^2 MB megabyte
+1000^3 GB gigabyte
+1000^4 TB terabyte
+1000^5 PB petabyte
+1000^6 EB exabyte
+
+useISU = False -> Binary (as defined by the International Electrotechnical Commission)
+Bytes / Metric
+1024^1 KiB kibibyte
+1024^2 MiB mebibyte
+1024^3 GiB gibibyte
+1024^4 TiB tebibyte
+1024^5 PiB pebibyte
+1024^6 EiB exbibyte
+*/
+func ByteSizeString(size int64, useISU bool) string {
+	var byteSize, unit float64 = float64(size), 1024
+	var pre string
+
+	if useISU {
+		unit = 1000
+	}
+
+	if byteSize < unit {
+		return fmt.Sprintf("%d B", int(byteSize))
+	}
+
+	exp := math.Floor(math.Log(byteSize) / math.Log(unit))
+
+	if useISU {
+		pre = string("kMGTPE"[int(exp-1)])
+	} else {
+		pre = fmt.Sprintf("%vi", string("KMGTPE"[int(exp-1)]))
+	}
+
+	res := byteSize / math.Pow(unit, exp)
+
+	return fmt.Sprintf("%.1f %sB", res, pre)
+}
+
+/*
+HexDump produces a more-or-less human readable hex dump from a given byte array
+slice.
+*/
+func HexDump(data []byte) string {
+	buf := new(bytes.Buffer)
+	line := new(bytes.Buffer)
+
+	buf.WriteString("====\n000000  ")
+
+	for i, b := range data {
+
+		if i != 0 && i%10 == 0 {
+			buf.WriteString(fmt.Sprintf(" %s\n%06x  ", line.String(), i))
+			line = new(bytes.Buffer)
+		}
+
+		buf.WriteString(fmt.Sprintf("%02X ", b))
+		line.WriteString(fmt.Sprintf("%c", b))
+	}
+
+	rest := len(data) % 10
+	if rest != 0 {
+		for i := rest; i < 10; i++ {
+			buf.WriteString("   ")
+		}
+	}
+
+	buf.WriteString(fmt.Sprintf(" %s\n====\n", line.String()))
+
+	return buf.String()
+}

+ 81 - 0
bitutil/bitutil_test.go

@@ -0,0 +1,81 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"testing"
+)
+
+func TestCompareByteArray(t *testing.T) {
+	testdata1 := []byte("Test")
+	testdata2 := make([]byte, 4, 5)
+	testdata3 := make([]byte, 3, 3)
+
+	if CompareByteArray(testdata1, testdata2) {
+		t.Error("Byte arrays should not be considered equal before copying data.")
+	}
+
+	if CompareByteArray(testdata1, testdata3) {
+		t.Error("Byte arrays should not be considered equal if the length is different.")
+	}
+
+	copy(testdata2, testdata1)
+
+	if cap(testdata1) == cap(testdata2) {
+		t.Error("Capacity of testdata sclices should be different.")
+	}
+
+	if !CompareByteArray(testdata1, testdata2) {
+		t.Error("Byte arrays should be considered equal.")
+	}
+}
+
+func TestByteSizeString(t *testing.T) {
+	// Test byte sizes
+	testdata := []int64{10000, 1024, 500, 1233456, 44166037, 84166037, 5000000000}
+
+	// non-ISU values
+	expected1 := []string{"9.8 KiB", "1.0 KiB", "500 B", "1.2 MiB", "42.1 MiB", "80.3 MiB", "4.7 GiB"}
+
+	// ISU values
+	expected2 := []string{"10.0 kB", "1.0 kB", "500 B", "1.2 MB", "44.2 MB", "84.2 MB", "5.0 GB"}
+
+	for i, test := range testdata {
+		res := ByteSizeString(test, false)
+		if res != expected1[i] {
+			t.Error("Unexpected value for non-isu value:", test,
+				"got:", res, "expected:", expected1[i])
+			return
+		}
+
+		res = ByteSizeString(test, true)
+		if res != expected2[i] {
+			t.Error("Unexpected value for isu value:", test,
+				"got:", res, "expected:", expected2[i])
+			return
+		}
+	}
+}
+
+func TestHexDump(t *testing.T) {
+	testdata := []byte("This is a test text. This is a test text.")
+
+	res := HexDump(testdata)
+	if res != "====\n"+
+		"000000  54 68 69 73 20 69 73 20 61 20  This is a \n"+
+		"00000a  74 65 73 74 20 74 65 78 74 2E  test text.\n"+
+		"000014  20 54 68 69 73 20 69 73 20 61   This is a\n"+
+		"00001e  20 74 65 73 74 20 74 65 78 74   test text\n"+
+		"000028  2E                             .\n"+
+		"====\n" {
+
+		t.Error("Invalid boundaries should cause an error")
+	}
+}

+ 89 - 0
bitutil/murmurhash3.go

@@ -0,0 +1,89 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import "fmt"
+
+const (
+	c1 uint32 = 0xcc9e2d51
+	c2 uint32 = 0x1b873593
+)
+
+/*
+MurMurHashData hashes a given array of bytes. This is an implementation
+of Austin Appleby's MurmurHash3 (32bit) function.
+
+Reference implementation: http://code.google.com/p/smhasher/wiki/MurmurHash3
+*/
+func MurMurHashData(data []byte, offset int, size int, seed int) (uint32, error) {
+
+	// Check parameters
+
+	if offset < 0 || size < 0 {
+		return 0, fmt.Errorf("Invalid data boundaries; offset: %v; size: %v",
+			offset, size)
+	}
+
+	h1 := uint32(seed)
+	end := offset + size
+	end -= end % 4
+
+	// Check length of available data
+
+	if len(data) <= end {
+		return 0, fmt.Errorf("Data out of bounds; set boundary: %v; data length: %v",
+			end, len(data))
+	}
+
+	for i := offset; i < end; i += 4 {
+
+		var k1 = uint32(data[i])
+		k1 |= uint32(data[i+1]) << 8
+		k1 |= uint32(data[i+2]) << 16
+		k1 |= uint32(data[i+3]) << 24
+
+		k1 *= c1
+		k1 = (k1 << 15) | (k1 >> 17) // ROTL32(k1,15);
+		k1 *= c2
+
+		h1 ^= k1
+		h1 = (h1 << 13) | (h1 >> 19) // ROTL32(h1,13);
+		h1 = h1*5 + 0xe6546b64
+	}
+
+	// Tail
+
+	var k1 uint32
+
+	switch size & 3 {
+	case 3:
+		k1 = uint32(data[end+2]) << 16
+		fallthrough
+	case 2:
+		k1 |= uint32(data[end+1]) << 8
+		fallthrough
+	case 1:
+		k1 |= uint32(data[end])
+		k1 *= c1
+		k1 = (k1 << 15) | (k1 >> 17) // ROTL32(k1,15);
+		k1 *= c2
+		h1 ^= k1
+	}
+
+	h1 ^= uint32(size)
+
+	h1 ^= h1 >> 16
+	h1 *= 0x85ebca6b
+	h1 ^= h1 >> 13
+	h1 *= 0xc2b2ae35
+	h1 ^= h1 >> 16
+
+	return h1, nil
+}

+ 75 - 0
bitutil/murmurhash3_test.go

@@ -0,0 +1,75 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"testing"
+)
+
+var testData = []byte("Now is the time for all good men to come to the aid of their country")
+
+var resultArray1 = []uint32{
+	0x249cb285, 0xcae32c45, 0x49cc6fdd, 0x3c89b814, 0xdc9778bb, 0x6db6607a,
+	0x736df8ad, 0xd367e257, 0x59b32232, 0x2496a9b4, 0x01d69f33, 0x08454378,
+	0x4ad4f630, 0x0ae1ca05, 0x042bdb5b, 0xbf3592e8, 0x0ed8b048, 0xb86958db,
+	0xa74ca5b6, 0xb7982271, 0x10a77c40, 0x8caba8ef, 0xe5085ab6, 0x8ee964b8,
+	0x170f0222, 0x42dec76d, 0xc4ebe4e5, 0x3d246566, 0x64f1133e, 0x8a0597dd,
+	0x5b13cdb8, 0x1c723636, 0xc8b60a2f, 0xb572fe46, 0xb801f177, 0x71d44c64,
+	0x755aeff1, 0x66ba2eeb, 0x5cfec249, 0x5b9d603f, 0x4e916049, 0x07622306,
+	0x57d4271f, 0x3fa8e56a, 0x4b4fe703, 0x995e958d, 0xdaf48fbb, 0xbe381e68,
+	0xd4af5452, 0x6b8e4cdc, 0x3c7bbc57, 0xd834a3e0, 0x78665c77, 0x5ab0d747,
+	0x4b34afb7, 0xbce90104, 0x25a31264, 0xa348c314, 0xab9fb213, 0x48f40ea9,
+	0xa232f18e, 0xda12f11a, 0x7dcdfcfb, 0x24381ba8, 0x1a15737d, 0x32b1ea01,
+	0x7ed7f6c6, 0xd16ab3ed}
+
+func TestMurMurHashData(t *testing.T) {
+
+	data := []byte{0xf6, 0x02, 0x03, 0x04}
+
+	// Test invalid data boundaries
+
+	_, err := MurMurHashData(data, 1, -3, 6)
+
+	if err == nil {
+		t.Error("Invalid boundaries should cause an error")
+	} else if err.Error() != "Invalid data boundaries; offset: 1; size: -3" {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	_, err = MurMurHashData(data, 1, 5, 6)
+
+	if err == nil {
+		t.Error("Invalid boundaries should cause an error")
+	} else if err.Error() != "Data out of bounds; set boundary: 4; data length: 4" {
+		t.Errorf("Unexpected error: %v", err)
+	}
+
+	// Test against data
+
+	// Go source code is always UTF-8, so the string literal is UTF-8 text.
+	data = []byte("Now is the time for all good men to come to the aid of their country")
+
+	doTest := func(offset, size int) uint32 {
+		res, err := MurMurHashData(data, offset, size, 4)
+
+		if err != nil {
+			t.Errorf("Unexpected error: %v", err)
+		}
+
+		return res
+	}
+
+	for i := 0; i < len(resultArray1); i++ {
+		res := doTest(0, i)
+		if res != resultArray1[i] {
+			t.Errorf("Unexpected result; Expected: 0x%x; Got: 0x%x", resultArray1[i], res)
+		}
+	}
+}

+ 416 - 0
bitutil/packedlist.go

@@ -0,0 +1,416 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"bytes"
+	"encoding/binary"
+	"math"
+)
+
+/*
+Different types of list packing
+*/
+const (
+	packListType2Bit = 0x1
+	packListType3Bit = 0x2
+	packListType6Bit = 0x3
+	packListTypeVar  = 0x0
+)
+
+/*
+PackList packs a given list to a string. Depending on the given highest number the
+list is packed in the most efficient way.
+*/
+func PackList(unpackedlist []uint64, highest uint64) string {
+
+	// Depending on the highest number convert to given list
+
+	switch {
+	case highest <= 3:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList2Bit(list)
+
+	case highest <= 7:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList3Bit(list)
+
+	case highest <= 63:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList6Bit(list)
+
+	case highest <= math.MaxUint8:
+		list := make([]byte, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = byte(num)
+		}
+		return PackList8Bit(list)
+
+	case highest <= math.MaxUint16:
+		list := make([]uint16, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = uint16(num)
+		}
+		return PackList16Bit(list)
+
+	case highest <= math.MaxUint32:
+		list := make([]uint32, len(unpackedlist))
+		for i, num := range unpackedlist {
+			list[i] = uint32(num)
+		}
+		return PackList32Bit(list)
+	}
+
+	return PackList64Bit(unpackedlist)
+}
+
+/*
+UnpackList unpacks a list from a packed string.
+*/
+func UnpackList(packedlist string) []uint64 {
+	plist := []byte(packedlist)
+
+	if len(plist) == 0 {
+		return nil
+	}
+
+	if plist[0]&0xC0 == packListTypeVar {
+		return UnpackBigList(packedlist)
+	}
+
+	res := UnpackSmallList(packedlist)
+	ret := make([]uint64, len(res))
+
+	for i, item := range res {
+		ret[i] = uint64(item)
+	}
+
+	return ret
+}
+
+/*
+PackList8Bit packs a list of 8 bit numbers.
+*/
+func PackList8Bit(list []uint8) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x00)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+PackList16Bit packs a list of 16 bit numbers.
+*/
+func PackList16Bit(list []uint16) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x01)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+PackList32Bit packs a list of 32 bit numbers.
+*/
+func PackList32Bit(list []uint32) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x02)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+PackList64Bit packs a list of 64 bit numbers.
+*/
+func PackList64Bit(list []uint64) string {
+	var bb bytes.Buffer
+
+	bb.WriteByte(0x03)
+
+	for i := 0; i < len(list); i++ {
+		binary.Write(&bb, binary.LittleEndian, list[i])
+	}
+
+	return bb.String()
+}
+
+/*
+UnpackBigList unpacks a list which has large values.
+*/
+func UnpackBigList(packedlist string) []uint64 {
+	var ret []uint64
+	plist := []byte(packedlist)
+
+	numlist := plist[1:]
+	reader := bytes.NewReader(numlist)
+
+	if plist[0] == 0x00 {
+		var item uint8
+		size := len(numlist)
+		ret = make([]uint64, size)
+		for i := 0; i < size; i++ {
+			binary.Read(reader, binary.LittleEndian, &item)
+			ret[i] = uint64(item)
+		}
+	} else if plist[0] == 0x01 {
+		var item uint16
+		size := len(numlist) / 2
+		ret = make([]uint64, size)
+		for i := 0; i < size; i++ {
+			binary.Read(reader, binary.LittleEndian, &item)
+			ret[i] = uint64(item)
+		}
+	} else if plist[0] == 0x02 {
+		var item uint32
+		size := len(numlist) / 4
+		ret = make([]uint64, size)
+		for i := 0; i < size; i++ {
+			binary.Read(reader, binary.LittleEndian, &item)
+			ret[i] = uint64(item)
+		}
+	} else if plist[0] == 0x03 {
+		size := len(numlist) / 8
+		ret = make([]uint64, size)
+		binary.Read(reader, binary.LittleEndian, ret)
+	}
+
+	return ret
+}
+
+/*
+PackList2Bit packs a list of bytes into a string using 2 bits for each item.
+(Items must be between 1 and 3)
+*/
+func PackList2Bit(list []byte) string {
+	if len(list) == 0 {
+		return ""
+	}
+
+	// Packing the list with 2 bit items reduces the size by a factor of 4
+
+	ret := make([]byte, int(math.Ceil(float64(1)/3+float64(len(list)-1)/4)))
+
+	if len(list) == 1 {
+		ret[0] = list2byte2bit(packListType2Bit, list[0], 0, 0)
+	} else if len(list) == 2 {
+		ret[0] = list2byte2bit(packListType2Bit, list[0], list[1], 0)
+	} else {
+		ret[0] = list2byte2bit(packListType2Bit, list[0], list[1], list[2])
+
+		j := 1
+		for i := 3; i < len(list); i += 4 {
+			if len(list[i:]) == 1 {
+				ret[j] = list2byte2bit(list[i], 0, 0, 0)
+			} else if len(list[i:]) == 2 {
+				ret[j] = list2byte2bit(list[i], list[i+1], 0, 0)
+			} else if len(list[i:]) == 3 {
+				ret[j] = list2byte2bit(list[i], list[i+1], list[i+2], 0)
+			} else {
+				ret[j] = list2byte2bit(list[i], list[i+1], list[i+2], list[i+3])
+			}
+			j++
+		}
+	}
+
+	return string(ret)
+}
+
+/*
+PackList3Bit packs a list of bytes into a string using 3 bits for each item.
+(Items must be between 1 and 7)
+*/
+func PackList3Bit(list []byte) string {
+	if len(list) == 0 {
+		return ""
+	}
+
+	// Packing the list with 2 bit items reduces the size by a factor of 2
+
+	ret := make([]byte, int(math.Ceil(float64(len(list))/2)))
+
+	if len(list) == 1 {
+		ret[0] = list2byte3bitAndHeader(packListType3Bit, list[0], 0)
+	} else {
+		ret[0] = list2byte3bitAndHeader(packListType3Bit, list[0], list[1])
+
+		j := 1
+		for i := 2; i < len(list); i += 2 {
+			if len(list[i:]) == 1 {
+				ret[j] = list2byte3bitAndHeader(0, list[i], 0)
+			} else {
+				ret[j] = list2byte3bitAndHeader(0, list[i], list[i+1])
+			}
+			j++
+		}
+	}
+
+	return string(ret)
+}
+
+/*
+PackList6Bit packs a list of bytes into a string using 6 bits for each item.
+(Items must be between 1 and 63)
+*/
+func PackList6Bit(list []byte) string {
+	if len(list) == 0 {
+		return ""
+	}
+
+	// Packing the list with 6 bit items does not reduce the factor
+
+	ret := make([]byte, len(list))
+
+	if len(list) == 1 {
+		ret[0] = list2byte6bitAndHeader(packListType6Bit, list[0])
+	} else {
+		ret[0] = list2byte6bitAndHeader(packListType6Bit, list[0])
+
+		for i := 1; i < len(list); i++ {
+			ret[i] = list2byte6bitAndHeader(0, list[i])
+		}
+	}
+
+	return string(ret)
+}
+
+/*
+UnpackSmallList unpacks a string into a list of bytes. Returns the list of bytes
+or a list of a single 0x00 byte if the numbers in the list are too big.
+*/
+func UnpackSmallList(packedlist string) []byte {
+	plist := []byte(packedlist)
+
+	if len(plist) == 0 {
+		return []byte{}
+	}
+
+	ltype := plist[0] & 0xC0 >> 6
+
+	if ltype == packListType2Bit {
+		return unpacklist2bit(plist)
+	} else if ltype == packListType3Bit {
+		return unpacklist3bit(plist)
+	} else if ltype == packListType6Bit {
+		return unpacklist6bit(plist)
+	}
+
+	// Must be gob encoded
+
+	return []byte{00}
+}
+
+func unpacklist2bit(packedlist []byte) []byte {
+	ret := make([]byte, 0, len(packedlist)*3)
+
+	for i := 0; i < len(packedlist); i++ {
+		b1, b2, b3, b4 := byte2list2bit(packedlist[i])
+		if i > 0 && b1 != 0 {
+			ret = append(ret, b1)
+		}
+		if b2 != 0 {
+			ret = append(ret, b2)
+		}
+		if b3 != 0 {
+			ret = append(ret, b3)
+		}
+		if b4 != 0 {
+			ret = append(ret, b4)
+		}
+	}
+
+	return ret
+}
+
+func unpacklist3bit(packedlist []byte) []byte {
+	ret := make([]byte, 0, len(packedlist)*2)
+
+	for i := 0; i < len(packedlist); i++ {
+		b1, b2 := byte2list3bit(packedlist[i])
+		if b1 != 0 {
+			ret = append(ret, b1)
+		}
+		if b2 != 0 {
+			ret = append(ret, b2)
+		}
+	}
+
+	return ret
+}
+
+func unpacklist6bit(packedlist []byte) []byte {
+	ret := make([]byte, 0, len(packedlist))
+
+	for i := 0; i < len(packedlist); i++ {
+		ret = append(ret, byte2list6bit(packedlist[i]))
+	}
+
+	return ret
+}
+
+func byte2list2bit(b byte) (b1 byte, b2 byte, b3 byte, b4 byte) {
+	b1 = b & 0xC0 >> 6
+	b2 = b & 0x30 >> 4
+	b3 = b & 0x0C >> 2
+	b4 = b & 0x03
+
+	return b1, b2, b3, b4
+}
+
+func list2byte2bit(b1 byte, b2 byte, b3 byte, b4 byte) byte {
+	return (b1 & 0x03 << 6) |
+		(b2 & 0x03 << 4) |
+		(b3 & 0x03 << 2) |
+		(b4 & 0x03)
+}
+
+func list2byte3bitAndHeader(b1 byte, b2 byte, b3 byte) byte {
+	return (b1 & 0x03 << 6) |
+		(b2 & 0x07 << 3) |
+		(b3 & 0x07)
+}
+
+func byte2list3bit(b byte) (b2 byte, b3 byte) {
+	b2 = b & 0x38 >> 3
+	b3 = b & 0x07
+
+	return b2, b3
+}
+
+func list2byte6bitAndHeader(b1 byte, b2 byte) byte {
+	return (b1 & 0x03 << 6) |
+		(b2 & 0x3F)
+}
+
+func byte2list6bit(b byte) byte {
+	return b & 0x3F
+}

+ 360 - 0
bitutil/packedlist_test.go

@@ -0,0 +1,360 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package bitutil
+
+import (
+	"fmt"
+	"math"
+	"testing"
+)
+
+func TestListPacking(t *testing.T) {
+	mylist := make([]uint64, 7)
+	mylist[0] = 3
+	mylist[1] = 7
+	mylist[2] = 63
+	mylist[3] = math.MaxUint8
+	mylist[4] = math.MaxUint16
+	mylist[5] = math.MaxUint32
+	mylist[6] = math.MaxUint64
+
+	res := UnpackList(PackList(mylist, 3))
+	if res[0] != 3 {
+		t.Error("Unexpected result:", res)
+		return
+	}
+
+	res = UnpackList(PackList(mylist, 7))
+	if fmt.Sprint(res[:2]) != "[3 7]" {
+		t.Error("Unexpected result:", res[:2])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, 63))
+	if fmt.Sprint(res[:3]) != "[3 7 63]" {
+		t.Error("Unexpected result:", res[:3])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint8))
+	if fmt.Sprint(res[:4]) != "[3 7 63 255]" {
+		t.Error("Unexpected result:", res[:4])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint16))
+	if fmt.Sprint(res[:5]) != "[3 7 63 255 65535]" {
+		t.Error("Unexpected result:", res[:5])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint32))
+	if fmt.Sprint(res[:6]) != "[3 7 63 255 65535 4294967295]" {
+		t.Error("Unexpected result:", res[:6])
+		return
+	}
+
+	res = UnpackList(PackList(mylist, math.MaxUint64))
+	if fmt.Sprint(res[:7]) != "[3 7 63 255 65535 4294967295 18446744073709551615]" {
+		t.Error("Unexpected result:", res[:7])
+		return
+	}
+
+	res = UnpackList(PackList([]uint64{10, 12, 80}, 80))
+	if fmt.Sprint(res) != "[10 12 80]" {
+		t.Error("Unexpected result:", res)
+		return
+	}
+}
+
+func TestListPacking8(t *testing.T) {
+	list1 := PackList3Bit([]byte{1, 2, 3, 4, 5, 6, 7})
+	list2 := PackList16Bit([]uint16{1, 2, 3, 4})
+
+	if len(list1) != 4 || len(list2) != 9 {
+		t.Error("Unexpected lengths:", len(list1), len(list2))
+		return
+	}
+
+	res1 := UnpackList(list1)
+	res2 := UnpackList(list2)
+
+	if fmt.Sprint(res1) != "[1 2 3 4 5 6 7]" {
+		t.Error("Unexpected result:", res1)
+		return
+	}
+	if fmt.Sprint(res2) != "[1 2 3 4]" {
+		t.Error("Unexpected result:", res2)
+		return
+	}
+
+	if UnpackList("") != nil {
+		t.Error("Unexpected result")
+		return
+	}
+}
+
+func TestVarBitListPacking8(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint8, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint8
+	}
+
+	res := PackList8Bit(testlist)
+
+	if len(res) != scale+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint8(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestVarBitListPacking16(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint16, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint16
+	}
+
+	res := PackList16Bit(testlist)
+
+	if len(res) != scale*2+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint16(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestVarBitListPacking32(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint32, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint32
+	}
+
+	res := PackList32Bit(testlist)
+
+	if len(res) != scale*4+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint32(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestVarBitListPacking64(t *testing.T) {
+	scale := 3
+
+	testlist := make([]uint64, scale)
+
+	for i := 0; i < scale; i++ {
+		testlist[i] = math.MaxUint64
+	}
+
+	res := PackList64Bit(testlist)
+
+	if len(res) != scale*8+1 {
+		t.Error("Unexpected length:", len(res))
+		return
+	}
+
+	res2 := UnpackBigList(res)
+
+	for i := 0; i < scale; i++ {
+		if testlist[i] != uint64(res2[i]) {
+			t.Error("Unexpected result at:", i)
+		}
+	}
+}
+
+func TestSmallListPacking(t *testing.T) {
+
+	// Test simple cases
+
+	if PackList2Bit([]byte{}) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if PackList3Bit([]byte{}) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if PackList6Bit([]byte{}) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if string(UnpackSmallList("")) != "" {
+		t.Error("Unexpected result")
+		return
+	}
+
+	// Simulates a gob encoded string
+
+	if string(UnpackSmallList(string([]byte{0x00}))) != string(0x00) {
+		t.Error("Unexpected result")
+		return
+	}
+
+	// Test normal cases
+
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 1, 2, 3}, []byte{0x5b, 0x6c}, 2)
+	checkListAndPresentation2bit(t, []byte{1}, []byte{0x50}, 1)
+	checkListAndPresentation2bit(t, []byte{1, 2}, []byte{0x58}, 1)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3}, []byte{0x5B}, 1)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 3}, []byte{0x5B, 0xC0}, 2)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 3, 2}, []byte{0x5B, 0xE0}, 2)
+	checkListAndPresentation2bit(t, []byte{1, 2, 3, 3, 2, 1, 3}, []byte{0x5B, 0xE7}, 2)
+
+	checkListAndPresentation3bit(t, []byte{1, 2, 3, 1, 2, 3}, []byte{0x8A, 0x19, 0x13}, 3)
+	checkListAndPresentation3bit(t, []byte{1}, []byte{0x88}, 1)
+	checkListAndPresentation3bit(t, []byte{1, 2}, []byte{0x8A}, 1)
+	checkListAndPresentation3bit(t, []byte{1, 2, 3}, []byte{0x8A, 0x18}, 2)
+	checkListAndPresentation3bit(t, []byte{1, 2, 3, 3}, []byte{0x8A, 0x1B}, 2)
+	checkListAndPresentation3bit(t, []byte{1, 2, 3, 4, 5, 6, 7}, []byte{0x8A, 0x1C, 0x2E, 0x38}, 4)
+
+	checkListAndPresentation6bit(t, []byte{1, 2, 3, 1, 2, 3})
+	checkListAndPresentation6bit(t, []byte{1})
+	checkListAndPresentation6bit(t, []byte{1, 2})
+	checkListAndPresentation6bit(t, []byte{1, 2, 3})
+	checkListAndPresentation6bit(t, []byte{1, 2, 3, 3})
+	checkListAndPresentation6bit(t, []byte{1, 2, 3, 4, 35, 45, 63})
+}
+
+func checkListAndPresentation2bit(t *testing.T, list []byte, packedlist []byte, packedLen int) {
+	res := PackList2Bit(list)
+	if res != string(packedlist) {
+		t.Errorf("Unexpected result: %X", []byte(res))
+		return
+	}
+	if len(res) != packedLen {
+		t.Error("Unexpected size", len(res))
+		return
+	}
+	if dres := UnpackSmallList(res); string(dres) != string(list) {
+		t.Errorf("Unexpected result: %X", []byte(dres))
+		return
+	}
+}
+
+func checkListAndPresentation3bit(t *testing.T, list []byte, packedlist []byte, packedLen int) {
+	res := PackList3Bit(list)
+	if res != string(packedlist) {
+		t.Errorf("Unexpected result: %X", []byte(res))
+		return
+	}
+	if len(res) != packedLen {
+		t.Error("Unexpected size", len(res))
+		return
+	}
+	if dres := UnpackSmallList(res); string(dres) != string(list) {
+		t.Errorf("Unexpected result: %X", []byte(dres))
+		return
+	}
+}
+
+func checkListAndPresentation6bit(t *testing.T, list []byte) {
+	res := PackList6Bit(list)
+
+	packedlist := make([]byte, len(list))
+	copy(packedlist, list)
+	packedlist[0] = packedlist[0] | 0xC0
+
+	if res != string(packedlist) {
+		t.Errorf("Unexpected result: %X vs %X", []byte(res), packedlist)
+		return
+	}
+	if len(res) != len(list) {
+		t.Error("Unexpected size", len(res))
+		return
+	}
+	if dres := UnpackSmallList(res); string(dres) != string(list) {
+		t.Errorf("Unexpected result: %X", []byte(dres))
+		return
+	}
+}
+
+func TestList2byte2bit(t *testing.T) {
+	if res := list2byte2bit(0x01, 0x2, 0x03, 0x01); res != 0x6D {
+		t.Errorf("Unexpected result: %X", res)
+		return
+	}
+	if res := list2byte3bitAndHeader(0x00, 0x07, 0x03); res != 0x3B {
+		t.Errorf("Unexpected result: %X", res)
+		return
+	}
+}
+
+func TestByte2list2bit(t *testing.T) {
+	if a, b, c, d := byte2list2bit(0x30); a != 00 || b != 03 || c != 00 || d != 00 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x80); a != 02 || b != 00 || c != 00 || d != 00 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x01); a != 00 || b != 00 || c != 00 || d != 01 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x31); a != 00 || b != 03 || c != 00 || d != 01 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+	if a, b, c, d := byte2list2bit(0x05); a != 00 || b != 00 || c != 01 || d != 01 {
+		t.Error("Unexpected result:", a, b, c, d)
+		return
+	}
+}
+
+func TestByte2list3bit(t *testing.T) {
+	if a, b := byte2list3bit(0x01); a != 00 || b != 01 {
+		t.Error("Unexpected result:", a, b)
+		return
+	}
+	if a, b := byte2list3bit(0x31); a != 06 || b != 01 {
+		t.Error("Unexpected result:", a, b)
+		return
+	}
+	if a, b := byte2list3bit(0x05); a != 00 || b != 05 {
+		t.Error("Unexpected result:", a, b)
+		return
+	}
+}

+ 189 - 0
cryptutil/gencert.go

@@ -0,0 +1,189 @@
+/*
+Package cryptutil contains cryptographic utility functions.
+
+Certificate generation code based on:
+go source src/crypto/tls/generate_cert.go
+
+Copyright 2009 The Go Authors. All rights reserved.
+Use of this source code is governed by a BSD-style license.
+*/
+package cryptutil
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"math/big"
+	"net"
+	"os"
+	"strings"
+	"time"
+)
+
+/*
+GenCert generates certificate files in a given path.
+
+path       - Path to generate the certificate in.
+certFile   - Certificate file to generate.
+keyFile    - Key file to generate.
+host       - Comma-separated hostnames and IPs to generate a certificate for.
+validFrom  - Creation date formatted as Jan 1 15:04:05 2011. Default is empty string which means now.
+validFor   - Duration that certificate is valid for. Default is 365*24*time.Hour.
+isCA       - Flag whether this cert should be its own Certificate Authority.
+rsaBits    - Size of RSA key to generate. Ignored if ecdsa-curve is set. Default is 2048.
+ecdsaCurve - ECDSA curve to use to generate a key. Valid values are P224, P256, P384, P521 or empty string (not set).
+*/
+func GenCert(path string, certFile string, keyFile string, host string,
+	validFrom string, validFor time.Duration, isCA bool, rsaBits int, ecdsaCurve string) error {
+
+	var err error
+
+	// Check parameters
+
+	if path != "" && !strings.HasSuffix(path, "/") {
+		path += "/"
+	}
+
+	if host == "" {
+		return errors.New("Host required for certificate generation")
+	}
+
+	var notBefore time.Time
+
+	if validFrom == "" {
+		notBefore = time.Now()
+	} else {
+		notBefore, err = time.Parse("Jan 2 15:04:05 2006", validFrom)
+		if err != nil {
+			return fmt.Errorf("Failed to parse creation date: %s", err)
+		}
+	}
+
+	notAfter := notBefore.Add(validFor)
+
+	// Generate private key
+
+	var priv interface{}
+
+	switch ecdsaCurve {
+	case "":
+		priv, err = rsa.GenerateKey(rand.Reader, rsaBits)
+	case "P224":
+		priv, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader)
+	case "P256":
+		priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	case "P384":
+		priv, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader)
+	case "P521":
+		priv, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader)
+	default:
+		err = fmt.Errorf("Unrecognized elliptic curve: %q", ecdsaCurve)
+	}
+
+	if err != nil {
+		return fmt.Errorf("Failed to generate private key: %s", err)
+	}
+
+	// Generate serial random number
+
+	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
+	serialNumber, _ := rand.Int(rand.Reader, serialNumberLimit)
+
+	// Create and populate the certificate template
+
+	template := x509.Certificate{
+
+		SerialNumber: serialNumber,
+		Subject: pkix.Name{
+			Organization: []string{"Local"},
+		},
+		NotBefore: notBefore,
+		NotAfter:  notAfter,
+
+		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+		BasicConstraintsValid: true,
+	}
+
+	// Add hosts
+
+	hosts := strings.Split(host, ",")
+	for _, h := range hosts {
+		if ip := net.ParseIP(h); ip != nil {
+			template.IPAddresses = append(template.IPAddresses, ip)
+		} else {
+			template.DNSNames = append(template.DNSNames, h)
+		}
+	}
+
+	// Set the CA flag
+
+	if isCA {
+		template.IsCA = isCA
+		template.KeyUsage |= x509.KeyUsageCertSign
+	}
+
+	// Create the certificate and write it out
+
+	derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv)
+
+	if err == nil {
+
+		certOut, err := os.Create(path + certFile)
+		defer certOut.Close()
+
+		if err != nil {
+			return fmt.Errorf("Failed to open %s for writing: %s", certFile, err)
+		}
+
+		pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
+
+		// Write out private key
+
+		keyOut, err := os.OpenFile(path+keyFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
+		defer keyOut.Close()
+
+		if err != nil {
+			return fmt.Errorf("Failed to open %v for writing: %v", keyFile, err)
+		}
+
+		pem.Encode(keyOut, pemBlockForKey(priv))
+	}
+
+	return err
+}
+
+/*
+Return public key from a given key pair.
+*/
+func publicKey(priv interface{}) interface{} {
+	switch k := priv.(type) {
+	case *rsa.PrivateKey:
+		return &k.PublicKey
+	case *ecdsa.PrivateKey:
+		return &k.PublicKey
+	default:
+		return nil
+	}
+}
+
+/*
+Return private key pem block for a given key pair.
+*/
+func pemBlockForKey(priv interface{}) *pem.Block {
+	switch k := priv.(type) {
+	case *rsa.PrivateKey:
+		return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)}
+	case *ecdsa.PrivateKey:
+		b, _ := x509.MarshalECPrivateKey(k)
+		return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b}
+	default:
+		return nil
+	}
+}

+ 149 - 0
cryptutil/gencert_test.go

@@ -0,0 +1,149 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"devt.de/krotik/common/fileutil"
+)
+
+const certDir = "certs"
+
+const invalidFileName = "**" + string(0x0)
+
+func TestMain(m *testing.M) {
+	flag.Parse()
+
+	// Setup
+	if res, _ := fileutil.PathExists(certDir); res {
+		os.RemoveAll(certDir)
+	}
+
+	err := os.Mkdir(certDir, 0770)
+	if err != nil {
+		fmt.Print("Could not create test directory:", err.Error())
+		os.Exit(1)
+	}
+
+	// Run the tests
+	res := m.Run()
+
+	// Teardown
+	err = os.RemoveAll(certDir)
+	if err != nil {
+		fmt.Print("Could not remove test directory:", err.Error())
+	}
+
+	os.Exit(res)
+}
+
+func TestGenCert(t *testing.T) {
+
+	checkGeneration := func(ecdsaCurve string) error {
+
+		// Generate a certificate and private key
+
+		err := GenCert(certDir, "cert.pem", "key.pem", "localhost,127.0.0.1", "", 365*24*time.Hour, true, 2048, ecdsaCurve)
+		if err != nil {
+			return err
+		}
+
+		// Check that the files were generated
+
+		if ok, _ := fileutil.PathExists(certDir + "/key.pem"); !ok {
+			return errors.New("Private key was not generated")
+		}
+
+		if ok, _ := fileutil.PathExists(certDir + "/cert.pem"); !ok {
+			return errors.New("Certificate was not generated")
+		}
+
+		_, err = ReadX509CertsFromFile(certDir + "/cert.pem")
+		if err != nil {
+			return err
+		}
+
+		return nil
+	}
+
+	if err := checkGeneration(""); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P224"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P256"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P384"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := checkGeneration("P521"); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Test error cases
+
+	err := GenCert(certDir, "cert.pem", "key.pem", "", "", 365*24*time.Hour, true, 2048, "")
+	if err.Error() != "Host required for certificate generation" {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, "cert.pem", "key.pem", "localhost", "", 365*24*time.Hour, true, 2048, "xxx")
+	if err.Error() != `Failed to generate private key: Unrecognized elliptic curve: "xxx"` {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, "cert.pem", "key.pem", "localhost", "xxx", 365*24*time.Hour, true, 2048, "")
+	if err.Error() != `Failed to parse creation date: parsing time "xxx" as "Jan 2 15:04:05 2006": cannot parse "xxx" as "Jan"` {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, "cert.pem", invalidFileName, "localhost", "", 365*24*time.Hour, true, 2048, "")
+	if !strings.HasPrefix(err.Error(), "Failed to open") {
+		t.Error(err)
+		return
+	}
+
+	err = GenCert(certDir, invalidFileName, "key.pem", "localhost", "", 365*24*time.Hour, true, 2048, "")
+	if !strings.HasPrefix(err.Error(), "Failed to open") {
+		t.Error(err)
+		return
+	}
+
+	if publicKey(nil) != nil {
+		t.Error("Unexpected result")
+		return
+	}
+
+	if pemBlockForKey(nil) != nil {
+		t.Error("Unexpected result")
+		return
+	}
+}

+ 109 - 0
cryptutil/stringcrypt.go

@@ -0,0 +1,109 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"io"
+)
+
+/*
+EncryptString encrypts a given string using AES (cfb mode).
+*/
+func EncryptString(passphrase, text string) (string, error) {
+	var ret []byte
+
+	// Create a new cipher with the given key
+
+	key := sha256.Sum256([]byte(passphrase))
+
+	block, err := aes.NewCipher((&key)[:])
+
+	if err == nil {
+
+		// Base64 encode the string
+
+		b := base64.StdEncoding.EncodeToString([]byte(text))
+
+		ciphertext := make([]byte, aes.BlockSize+len(b))
+
+		// Create the initialization vector using random numbers
+
+		iv := ciphertext[:aes.BlockSize]
+
+		if _, err = io.ReadFull(rand.Reader, iv); err == nil {
+
+			// Do the encryption
+
+			cfb := cipher.NewCFBEncrypter(block, iv)
+
+			cfb.XORKeyStream(ciphertext[aes.BlockSize:], []byte(b))
+
+			ret = ciphertext
+		}
+	}
+
+	return string(ret), err
+}
+
+/*
+DecryptString decrypts a given string using AES (cfb mode).
+*/
+func DecryptString(passphrase, text string) (string, error) {
+	var ret []byte
+
+	// Check encrypted text
+
+	if len(text) < aes.BlockSize {
+		return "", fmt.Errorf("Ciphertext is too short - must be at least: %v", aes.BlockSize)
+	}
+
+	// Create a new cipher with the given key
+
+	key := sha256.Sum256([]byte(passphrase))
+
+	block, err := aes.NewCipher((&key)[:])
+
+	if err == nil {
+
+		// Separate initialization vector and actual encrypted text
+
+		iv := text[:aes.BlockSize]
+
+		text = text[aes.BlockSize:]
+
+		// Do the decryption
+
+		cfb := cipher.NewCFBDecrypter(block, []byte(iv))
+
+		ret = []byte(text) // Reuse text buffer
+
+		cfb.XORKeyStream(ret, []byte(text))
+
+		// Decode text from base64
+
+		ret, err = base64.StdEncoding.DecodeString(string(ret))
+
+		if err != nil {
+
+			// Return a proper error if something went wrong
+
+			ret = nil
+			err = fmt.Errorf("Could not decrypt data")
+		}
+	}
+
+	return string(ret), err
+}

+ 53 - 0
cryptutil/stringcrypt_test.go

@@ -0,0 +1,53 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"testing"
+)
+
+func TestStringEncryption(t *testing.T) {
+
+	secret := "This is a test"
+
+	encString, err := EncryptString("foo", secret)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	decString, err := DecryptString("foo", encString)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if decString != secret {
+		t.Error("Unexpected result:", decString, secret)
+		return
+	}
+
+	decString, err = DecryptString("foo1", encString)
+	if err.Error() != "Could not decrypt data" {
+		t.Error(err)
+		return
+	}
+
+	if decString != "" {
+		t.Error("Unexpected result:", decString)
+		return
+	}
+
+	decString, err = DecryptString("foo1", "bar")
+	if err.Error() != "Ciphertext is too short - must be at least: 16" {
+		t.Error(err)
+		return
+	}
+}

+ 37 - 0
cryptutil/uuid.go

@@ -0,0 +1,37 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"crypto/rand"
+
+	"devt.de/krotik/common/errorutil"
+)
+
+/*
+GenerateUUID generates a version 4 (randomly generated) UUID according to RFC4122.
+*/
+func GenerateUUID() [16]byte {
+	var u [16]byte
+
+	_, err := rand.Read(u[:])
+	errorutil.AssertOk(err)
+
+	// Set version 4
+
+	u[6] = (u[6] & 0x0f) | 0x40
+
+	// Set variant bits - variant of RFC 4122
+
+	u[8] = (u[8] & 0xbf) | 0x80
+
+	return u
+
+}

+ 22 - 0
cryptutil/uuid_test.go

@@ -0,0 +1,22 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"fmt"
+	"testing"
+)
+
+func TestUUID(t *testing.T) {
+	if fmt.Sprint(GenerateUUID()) == "" {
+		t.Error("Unexpected result")
+		return
+	}
+}

+ 109 - 0
cryptutil/x509util.go

@@ -0,0 +1,109 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"bytes"
+	"crypto/md5"
+	"crypto/sha1"
+	"crypto/sha256"
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+)
+
+/*
+ReadX509CertsFromFile reads a list of pem encoded certificates from a given file.
+*/
+func ReadX509CertsFromFile(filename string) ([]*x509.Certificate, error) {
+	var err error
+	var certs []*x509.Certificate
+
+	file, err := os.OpenFile(filename, os.O_RDONLY, 0660)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	certsString, err := ioutil.ReadAll(file)
+	if err == nil {
+		certs, err = ReadX509Certs(certsString)
+	}
+
+	return certs, err
+}
+
+/*
+ReadX509Certs reads a list of pem encoded certificates from a byte array.
+*/
+func ReadX509Certs(certs []byte) ([]*x509.Certificate, error) {
+
+	var blocks []byte
+
+	for {
+		var block *pem.Block
+
+		block, certs = pem.Decode(certs)
+		if block == nil {
+			return nil, errors.New("PEM not parsed")
+		}
+
+		blocks = append(blocks, block.Bytes...)
+		if len(certs) == 0 {
+			break
+		}
+	}
+	c, err := x509.ParseCertificates(blocks)
+	if err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+/*
+Sha1CertFingerprint computes a sha1 fingerprint for a certificate.
+*/
+func Sha1CertFingerprint(cert *x509.Certificate) string {
+	return formatFingerprint(fmt.Sprintf("%x", sha1.Sum(cert.Raw)))
+}
+
+/*
+Sha256CertFingerprint computes a sha256 fingerprint for a certificate.
+*/
+func Sha256CertFingerprint(cert *x509.Certificate) string {
+	return formatFingerprint(fmt.Sprintf("%x", sha256.Sum256(cert.Raw)))
+}
+
+/*
+Md5CertFingerprint computes a md5 fingerprint for a certificate.
+*/
+func Md5CertFingerprint(cert *x509.Certificate) string {
+	return formatFingerprint(fmt.Sprintf("%x", md5.Sum(cert.Raw)))
+}
+
+/*
+Format a given fingerprint string.
+*/
+func formatFingerprint(raw string) string {
+	var buf bytes.Buffer
+
+	for i, c := range raw {
+		buf.WriteByte(byte(c))
+		if (i+1)%2 == 0 && i != len(raw)-1 {
+			buf.WriteByte(byte(':'))
+		}
+	}
+
+	return buf.String()
+}

+ 94 - 0
cryptutil/x509util_test.go

@@ -0,0 +1,94 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package cryptutil
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestCertificateDecoding(t *testing.T) {
+
+	_, err := ReadX509CertsFromFile(invalidFileName)
+	if err == nil {
+		t.Error("Attempting to load an invalid file should result in an error")
+		return
+	}
+
+	googleCert := `
+-----BEGIN CERTIFICATE-----
+MIIEgDCCA2igAwIBAgIIORWTXMrZJggwDQYJKoZIhvcNAQELBQAwSTELMAkGA1UE
+BhMCVVMxEzARBgNVBAoTCkdvb2dsZSBJbmMxJTAjBgNVBAMTHEdvb2dsZSBJbnRl
+cm5ldCBBdXRob3JpdHkgRzIwHhcNMTYwNzEzMTMxODU2WhcNMTYxMDA1MTMxNjAw
+WjBoMQswCQYDVQQGEwJVUzETMBEGA1UECAwKQ2FsaWZvcm5pYTEWMBQGA1UEBwwN
+TW91bnRhaW4gVmlldzETMBEGA1UECgwKR29vZ2xlIEluYzEXMBUGA1UEAwwOd3d3
+Lmdvb2dsZS5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDkNYMd
+9AGxMuv6wC7XBkzi6G7l+jqq+xoxs3zW+8jmGntRh/ggnTNLTQiwLPquusGbPo4n
+bVX2UQV7ATyWeg7WZQuVjgeeF7WG++xwtLUtW3noSCmePSasWx0mcJu2tiuMWqsm
+PbR08k14tz4jiqmRDQQfttffVS1wk0Oul6+x7hN8AyZ24gUWzb+L5ILA+8CtsZB/
+u9XFtf+yEr277J7vH7GyEJxYt3u2dxy/nrNlF8o2wUl+U1bvUnQVRPNiFXLK2uiQ
+4XkL7F3Uk19q09snjHcOixYHSYgyGYATCfV/d6hQ+RSKzd7TQp/YHtT1LgmUUefH
+Hu04LXVnuhKUYYZnAgMBAAGjggFLMIIBRzAdBgNVHSUEFjAUBggrBgEFBQcDAQYI
+KwYBBQUHAwIwGQYDVR0RBBIwEIIOd3d3Lmdvb2dsZS5jb20waAYIKwYBBQUHAQEE
+XDBaMCsGCCsGAQUFBzAChh9odHRwOi8vcGtpLmdvb2dsZS5jb20vR0lBRzIuY3J0
+MCsGCCsGAQUFBzABhh9odHRwOi8vY2xpZW50czEuZ29vZ2xlLmNvbS9vY3NwMB0G
+A1UdDgQWBBRU6a8Q+y3AwMTsYpTXqT+xJ6n9bzAMBgNVHRMBAf8EAjAAMB8GA1Ud
+IwQYMBaAFErdBhYbvPZotXb1gba7Yhq6WoEvMCEGA1UdIAQaMBgwDAYKKwYBBAHW
+eQIFATAIBgZngQwBAgIwMAYDVR0fBCkwJzAloCOgIYYfaHR0cDovL3BraS5nb29n
+bGUuY29tL0dJQUcyLmNybDANBgkqhkiG9w0BAQsFAAOCAQEAiw4H269LfRl/Vrm6
+BmTCS5ipvbE6qMbwdB++eA/NaHU29bbFzRIRIo7T6nHynAE6QTUS0fRoZ2bnoaxY
+Z98hSqnPlpDC3D2IImcrSywIejS0aFcT6UZT57QUm7iANDs3N7XHsXXLT0wrvXZS
+GPKxS2JtOS3J5lRoN4fbYLuAHEzBn7zAqtrd98EEaYGdDerMo8kAyIDHqV4OiukI
+YkefRqQpi1B8hPFuFw8KDGuAHdfHOoUmuRo4yxs5Br7FhoLLtdN+5UD3tbWYGZo4
+9dl+K2ZqYOiNIHSTg78YaLM2s82G0WcL3oSzZg/ne+HZdhTu2YNFbGnoBIrgPjiP
+TV6Wsg==
+-----END CERTIFICATE-----
+`
+
+	c, err := ReadX509Certs([]byte(googleCert))
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	if len(c) != 1 {
+		t.Error("Only one certificate should have been read")
+		return
+	}
+
+	if res := Sha256CertFingerprint(c[0]); res != "d0:88:88:3c:7b:b3:da:b4:9e:d8:bf:ec:43:aa:92:cb:29:58:e8:e2:e1:c3:89:8d:73:50:6a:b8:c8:f1:12:21" {
+		t.Error("Unexpected fingerprint:", res)
+		return
+	}
+
+	if res := Sha1CertFingerprint(c[0]); res != "ee:b6:d4:d8:88:e5:75:5f:ff:c0:19:27:b6:67:9c:77:e8:0d:2c:7f" {
+		t.Error("Unexpected fingerprint:", res)
+		return
+	}
+
+	if res := Md5CertFingerprint(c[0]); res != "5c:a6:bd:96:9c:96:79:a7:90:ee:89:a6:ee:1a:04:a8" {
+		t.Error("Unexpected fingerprint:", res)
+		return
+	}
+
+	// Test error cases
+
+	_, err = ReadX509Certs([]byte(googleCert[2:]))
+	if err.Error() != "PEM not parsed" {
+		t.Error("PEM parsing error expected:", err)
+		return
+	}
+
+	_, err = ReadX509Certs([]byte(googleCert[0:29] + "Mi" + googleCert[31:]))
+	if strings.HasPrefix("asn1: structure error", err.Error()) {
+		t.Error("asn1 parsing error expected:", err)
+		return
+	}
+}

+ 65 - 0
datautil/datacopy.go

@@ -0,0 +1,65 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+/*
+Package datautil contains general data handling objects and helper methods.
+*/
+package datautil
+
+import (
+	"bytes"
+	"encoding/gob"
+
+	"devt.de/krotik/common/pools"
+)
+
+/*
+bufferPool holds buffers which are used to copy objects.
+*/
+var bufferPool = pools.NewByteBufferPool()
+
+/*
+CopyObject copies contents of a given object reference to another given object reference.
+*/
+func CopyObject(src interface{}, dest interface{}) error {
+	bb := bufferPool.Get().(*bytes.Buffer)
+
+	err := gob.NewEncoder(bb).Encode(src)
+
+	if err != nil {
+		return err
+	}
+
+	err = gob.NewDecoder(bb).Decode(dest)
+
+	if err != nil {
+		return err
+	}
+
+	bb.Reset()
+	bufferPool.Put(bb)
+
+	return nil
+}
+
+/*
+MergeMaps merges all given maps into a new map. Contents are shallow copies
+and conflicts are resolved as last-one-wins.
+*/
+func MergeMaps(maps ...map[string]interface{}) map[string]interface{} {
+	ret := make(map[string]interface{})
+
+	for _, m := range maps {
+		for k, v := range m {
+			ret[k] = v
+		}
+	}
+
+	return ret
+}

+ 80 - 0
datautil/datacopy_test.go

@@ -0,0 +1,80 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import (
+	"testing"
+
+	"devt.de/krotik/common/testutil"
+)
+
+func TestMergeMaps(t *testing.T) {
+	m := MergeMaps(map[string]interface{}{
+		"a": 1,
+		"b": 2,
+	}, map[string]interface{}{
+		"b": 3,
+		"c": 4,
+	})
+
+	if len(m) != 3 {
+		t.Error("Unexpected number of result entries:", len(m))
+		return
+	}
+
+	if m["a"] != 1 || m["b"] != 3 || m["c"] != 4 {
+		t.Error("Unexpected entries:", m)
+		return
+	}
+}
+
+func TestCopyObject(t *testing.T) {
+
+	var ret2 string
+
+	if err := CopyObject("test", &ret2); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Test encoding errors
+
+	var ret3 testutil.GobTestObject
+
+	gobtest := &testutil.GobTestObject{Name: "test", EncErr: true, DecErr: false}
+
+	if err := CopyObject(gobtest, &ret3); err == nil || err.Error() != "Encode error" {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	gobtest = &testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false}
+	ret3 = testutil.GobTestObject{Name: "test", EncErr: false, DecErr: true}
+
+	if err := CopyObject(gobtest, &ret3); err == nil || err.Error() != "Decode error" {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	ret3 = testutil.GobTestObject{Name: "test", EncErr: true, DecErr: false}
+
+	if err := CopyObject(&ret3, gobtest); err == nil || err.Error() != "Encode error" {
+		t.Error("Unexpected result:", err)
+		return
+	}
+
+	ret3 = testutil.GobTestObject{Name: "test", EncErr: false, DecErr: false}
+	gobtest = &testutil.GobTestObject{Name: "test", EncErr: false, DecErr: true}
+
+	if err := CopyObject(&ret3, gobtest); err == nil || err.Error() != "Decode error" {
+		t.Error("Unexpected result:", err)
+		return
+	}
+}

+ 246 - 0
datautil/mapcache.go

@@ -0,0 +1,246 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import (
+	"bytes"
+	"fmt"
+	"math"
+	"sort"
+	"sync"
+	"time"
+)
+
+/*
+MapCache is a map based cache object storing string->interface{}. It is possible
+to specify a maximum size, which when reached causes the oldest entries to be
+removed. It is also possible to set an expiry time for values. Values which are
+old are purged on the next access to the object.
+*/
+type MapCache struct {
+	data    map[string]interface{} // Data for the cache
+	ts      map[string]int64       // Timestamps for values
+	size    uint64                 // Size of the cache
+	maxsize uint64                 // Max size of the cache
+	maxage  int64                  // Max age of the cache
+	mutex   *sync.RWMutex          // Mutex to protect atomic map operations
+}
+
+/*
+NewMapCache creates a new MapCache object. The calling function can specify
+the maximum size and the maximum age in seconds for entries. A value of 0
+means no size constraint and no age constraint.
+*/
+func NewMapCache(maxsize uint64, maxage int64) *MapCache {
+	return &MapCache{make(map[string]interface{}), make(map[string]int64),
+		0, maxsize, maxage, &sync.RWMutex{}}
+}
+
+/*
+Clear removes all entries.
+*/
+func (mc *MapCache) Clear() {
+
+	// Take writer lock
+
+	mc.mutex.Lock()
+	defer mc.mutex.Unlock()
+
+	mc.data = make(map[string]interface{})
+	mc.ts = make(map[string]int64)
+
+	mc.size = 0
+}
+
+/*
+Size returns the current size of the MapCache.
+*/
+func (mc *MapCache) Size() uint64 {
+	return mc.size
+}
+
+/*
+Put stores an item in the MapCache.
+*/
+func (mc *MapCache) Put(k string, v interface{}) {
+
+	// Do cache maintenance
+
+	oldest := mc.maintainCache()
+
+	// Take writer lock
+
+	mc.mutex.Lock()
+	defer mc.mutex.Unlock()
+
+	// Check if the entry is a new entry
+
+	if _, exists := mc.data[k]; !exists {
+
+		// If the list is full remove the oldest item otherwise increase the size
+
+		if mc.maxsize != 0 && mc.size == mc.maxsize {
+			delete(mc.data, oldest)
+			delete(mc.ts, oldest)
+		} else {
+			mc.size++
+		}
+	}
+
+	// Do the actual map operation
+
+	mc.data[k] = v
+	mc.ts[k] = time.Now().Unix()
+}
+
+/*
+Remove removes an item in the MapCache.
+*/
+func (mc *MapCache) Remove(k string) bool {
+
+	// Do cache maintenance
+
+	mc.maintainCache()
+
+	// Take writer lock
+
+	mc.mutex.Lock()
+	defer mc.mutex.Unlock()
+
+	// Check if the entry exists
+
+	_, exists := mc.data[k]
+
+	if exists {
+
+		// Do the actual map operation
+
+		delete(mc.data, k)
+		delete(mc.ts, k)
+
+		mc.size--
+	}
+
+	return exists
+}
+
+/*
+Get retrieves an item from the MapCache.
+*/
+func (mc *MapCache) Get(k string) (interface{}, bool) {
+
+	// Do cache maintenance
+
+	mc.maintainCache()
+
+	// Take reader lock
+
+	mc.mutex.RLock()
+	defer mc.mutex.RUnlock()
+
+	// Do the actual map operation
+
+	v, ok := mc.data[k]
+
+	return v, ok
+}
+
+/*
+GetAll retrieves all items from the MapCache.
+*/
+func (mc *MapCache) GetAll() map[string]interface{} {
+
+	// Do cache maintenance
+
+	mc.maintainCache()
+
+	// Take reader lock
+
+	mc.mutex.RLock()
+	defer mc.mutex.RUnlock()
+
+	// Create return map
+
+	cp := make(map[string]interface{})
+
+	for k, v := range mc.data {
+		cp[k] = v
+	}
+
+	return cp
+}
+
+/*
+String returns a string representation of this MapCache.
+*/
+func (mc *MapCache) String() string {
+
+	mc.mutex.RLock()
+	defer mc.mutex.RUnlock()
+
+	// Sort keys before printing the map
+
+	var keys []string
+	for k := range mc.data {
+		keys = append(keys, k)
+	}
+	sort.Sort(sort.StringSlice(keys))
+
+	buf := &bytes.Buffer{}
+	for _, k := range keys {
+		buf.WriteString(fmt.Sprint(k, ":", mc.data[k], "\n"))
+	}
+
+	return buf.String()
+}
+
+/*
+maintainCache removes expired items and returns the oldest entry.
+*/
+func (mc *MapCache) maintainCache() string {
+
+	mc.mutex.RLock()
+
+	oldestTS := int64(math.MaxInt64)
+	oldestK := ""
+
+	now := time.Now().Unix()
+
+	for k, v := range mc.ts {
+
+		// Check if the entry has expired
+
+		if mc.maxage != 0 && now-v > mc.maxage {
+
+			// Remove entry if it has expired
+
+			mc.mutex.RUnlock()
+			mc.mutex.Lock()
+
+			delete(mc.data, k)
+			delete(mc.ts, k)
+			mc.size--
+
+			mc.mutex.Unlock()
+			mc.mutex.RLock()
+		}
+
+		// Gather oldest entry
+
+		if v < oldestTS {
+			oldestTS = v
+			oldestK = k
+		}
+	}
+
+	mc.mutex.RUnlock()
+
+	return oldestK
+}

+ 148 - 0
datautil/mapcache_test.go

@@ -0,0 +1,148 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import (
+	"testing"
+	"time"
+)
+
+func TestMapCache(t *testing.T) {
+
+	// Create a map cache which can hold a maximum of 3 items for no longer than
+	// 5 seconds
+
+	mc := NewMapCache(3, 5)
+
+	mc.Put("k1", "aaa")
+	mc.Put("k2", "bbb")
+	mc.Put("k3", "ccc")
+
+	if s := mc.Size(); s != 3 {
+		t.Error("Unexpected size:", s)
+		return
+	}
+
+	mc.Clear()
+
+	if s := mc.Size(); s != 0 {
+		t.Error("Unexpected size:", s)
+		return
+	}
+
+	mc.Put("k1", "aaa")
+	mc.Put("k2", "bbb")
+	mc.Put("k3", "ccc")
+
+	if s := mc.Size(); s != 3 {
+		t.Error("Unexpected size:", s)
+		return
+	}
+
+	// Test copy
+
+	cp := mc.GetAll()
+
+	if len(cp) != 3 {
+		t.Error("Unexpected copy result:", cp)
+		return
+	}
+
+	// Simulate different timings
+
+	mc.ts["k1"] = time.Now().Unix() - 6 // Expired
+	mc.ts["k2"] = time.Now().Unix() - 3 // Oldest entry
+
+	if mc.String() != `
+k1:aaa
+k2:bbb
+k3:ccc
+`[1:] {
+		t.Error("Unexpected cache content:", mc)
+		return
+	}
+
+	// Do a read operation on an expired entry
+
+	if e, ok := mc.Get("k1"); e != nil || ok {
+		t.Error("Expired entry should not be returned", ok, e)
+		return
+	}
+
+	if mc.String() != `
+k2:bbb
+k3:ccc
+`[1:] {
+		t.Error("Unexpected cache content:", mc)
+		return
+	}
+
+	// Do a read operation on a live entry
+
+	if e, ok := mc.Get("k2"); e != "bbb" || !ok {
+		t.Error("Live entry should be returned", ok, e)
+		return
+	}
+
+	if mc.String() != `
+k2:bbb
+k3:ccc
+`[1:] {
+		t.Error("Unexpected cache content:", mc)
+		return
+	}
+
+	// Add 1 entry and update another
+
+	mc.Put("k3", "updateccc")
+	mc.Put("k4", "ddd")
+
+	if mc.String() != `
+k2:bbb
+k3:updateccc
+k4:ddd
+`[1:] {
+		t.Error("Unexpected cache content:", mc)
+		return
+	}
+
+	// Add another entry which should push out the oldest
+
+	mc.Put("k5", "eee")
+
+	if mc.String() != `
+k3:updateccc
+k4:ddd
+k5:eee
+`[1:] {
+		t.Error("Unexpected cache content:", mc)
+		return
+	}
+
+	// Remove items
+
+	if !mc.Remove("k3") {
+		t.Error("Live item should be deleted")
+		return
+	}
+
+	if mc.String() != `
+k4:ddd
+k5:eee
+`[1:] {
+		t.Error("Unexpected cache content:", mc)
+		return
+	}
+
+	if mc.Remove("k0") {
+		t.Error("Removal of non-existing item should not return success")
+		return
+	}
+}

+ 52 - 0
datautil/nesting.go

@@ -0,0 +1,52 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import "fmt"
+
+/*
+GetNestedValue gets a value from a nested object structure.
+*/
+func GetNestedValue(d map[string]interface{}, path []string) (interface{}, error) {
+	var ret interface{}
+	var err error
+
+	getNestedMap := func(d map[string]interface{}, key string) (map[string]interface{}, error) {
+		val := d[key]
+		newMap, ok := val.(map[string]interface{})
+
+		if !ok {
+			return nil, fmt.Errorf("Unexpected data type %T as value of %v", val, key)
+		}
+
+		return newMap, nil
+	}
+
+	// Drill into the object structure and return the requested value.
+
+	nestedMap := d
+	atomLevel := len(path) - 1
+
+	for i, elem := range path {
+
+		if i < atomLevel {
+
+			if nestedMap, err = getNestedMap(nestedMap, elem); err != nil {
+				break
+			}
+
+		} else {
+
+			ret = nestedMap[elem]
+		}
+	}
+
+	return ret, err
+}

+ 79 - 0
datautil/nesting_test.go

@@ -0,0 +1,79 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"testing"
+)
+
+func TestNesting(t *testing.T) {
+
+	// Create a nested piece of data which is serialized and deserialized
+
+	var testData1 = map[string]interface{}{
+		"level1": map[string]interface{}{
+			"level2": map[string]interface{}{
+				"atom": 42,
+			},
+			"atom2": "test5",
+		},
+	}
+
+	var bb1 bytes.Buffer
+
+	// Only register the generic map[string]interface{}
+
+	gob.Register(map[string]interface{}{})
+
+	if err := gob.NewEncoder(&bb1).Encode(testData1); err != nil {
+		t.Error(err)
+		return
+	}
+
+	var testOut map[string]interface{}
+
+	if err := gob.NewDecoder(&bb1).Decode(&testOut); err != nil {
+		t.Error(err)
+		return
+	}
+
+	val, err := GetNestedValue(testOut, []string{"level1", "level2", "atom"})
+	if val != 42 || err != nil {
+		t.Error("Unexpected result:", val, err)
+		return
+	}
+
+	val, err = GetNestedValue(testOut, []string{"level1", "level2"})
+	if fmt.Sprint(val) != "map[atom:42]" || err != nil {
+		t.Error("Unexpected result:", val, err)
+		return
+	}
+
+	val, err = GetNestedValue(testOut, []string{"level1", "atom2"})
+	if val != "test5" || err != nil {
+		t.Error("Unexpected result:", val, err)
+		return
+	}
+
+	val, err = GetNestedValue(testOut, []string{"level1", "atom3"})
+	if val != nil || err != nil {
+		t.Error("Unexpected result:", val, err)
+		return
+	}
+
+	val, err = GetNestedValue(testOut, []string{"level1", "level2", "atom", "test"})
+	if val != nil || err.Error() != "Unexpected data type int as value of atom" {
+		t.Error("Unexpected result:", val, err)
+		return
+	}
+}

+ 102 - 0
datautil/nonce.go

@@ -0,0 +1,102 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import (
+	"crypto/sha256"
+	"errors"
+	"fmt"
+
+	"devt.de/krotik/common/cryptutil"
+	"devt.de/krotik/common/timeutil"
+)
+
+/*
+MaxNonceLifetime is the maximum lifetime for nonces in seconds.
+*/
+var MaxNonceLifetime int64 = 3600 // One hour
+
+/*
+Default nonce related errors
+*/
+var (
+	ErrInvlaidNonce = errors.New("Invalid nonce value")
+)
+
+/*
+nonces is an internal map which holds all valid nonces
+*/
+var nonces *MapCache
+
+/*
+NewNonce generates a new nonce value. The nonce is invalidated either
+after it was consumed or automatically after MaxNonceLifetime seconds.
+*/
+func NewNonce() string {
+
+	if nonces == nil {
+
+		// Create nonce cache if it doesn't exist yet
+
+		nonces = NewMapCache(0, MaxNonceLifetime)
+	}
+
+	// Get a timestamp
+
+	ts := timeutil.MakeTimestamp()
+
+	// Calculate a hash based on a UUID
+
+	uuid := cryptutil.GenerateUUID()
+	secPart := sha256.Sum256(uuid[:])
+
+	// Construct the actual nonce and save it
+
+	ret := fmt.Sprintf("%x-%s", secPart, ts)
+
+	nonces.Put(ret, nil)
+
+	return ret
+}
+
+/*
+CheckNonce checks if a given nonce is valid. The nonce is still valid
+after this operation.
+*/
+func CheckNonce(nonce string) error {
+
+	// Check length
+
+	if len(nonce) == 78 && nonces != nil {
+
+		// Check if the nonce is still valid
+
+		if _, ok := nonces.Get(nonce); ok {
+			return nil
+		}
+	}
+
+	return ErrInvlaidNonce
+}
+
+/*
+ConsumeNonce consumes a given nonce. The nonce will no longer be valid
+after this operation.
+*/
+func ConsumeNonce(nonce string) error {
+
+	err := CheckNonce(nonce)
+
+	if err == nil {
+		nonces.Remove(nonce)
+	}
+
+	return nil
+}

+ 55 - 0
datautil/nonce_test.go

@@ -0,0 +1,55 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import (
+	"testing"
+)
+
+func TestNonces(t *testing.T) {
+
+	n1 := NewNonce()
+	n2 := NewNonce()
+
+	// Test normal check
+
+	if err := CheckNonce(n1); err != nil {
+		t.Error(err)
+		return
+	}
+
+	// Test consumption
+
+	if err := ConsumeNonce(n1); err != nil {
+		t.Error(err)
+		return
+	}
+
+	if err := CheckNonce(n1); err != ErrInvlaidNonce {
+		t.Error("Nonce should no longer be valid")
+		return
+	}
+
+	// Simulate timeout
+
+	nonces = nil
+
+	if err := CheckNonce(n2); err != ErrInvlaidNonce {
+		t.Error("Nonce should no longer be valid")
+		return
+	}
+
+	// Test error case
+
+	if err := CheckNonce("test"); err != ErrInvlaidNonce {
+		t.Error("Nonce should no longer be valid")
+		return
+	}
+}

+ 112 - 0
datautil/persistentmap.go

@@ -0,0 +1,112 @@
+/*
+ * Public Domain Software
+ *
+ * I (Matthias Ladkau) am the author of the source code in this file.
+ * I have placed the source code in this file in the public domain.
+ *
+ * For further information see: http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+package datautil
+
+import (
+	"encoding/gob"
+	"os"
+)
+
+/*
+PersistentMap is a persistent map storing string values. This implementation returns
+more encoding / decoding errors since not all possible values are supported.
+*/
+type PersistentMap struct {
+	filename string                 // File of the persistent map
+	Data     map[string]interface{} // Data of the persistent map
+}
+
+/*
+NewPersistentMap creates a new persistent map.
+*/
+func NewPersistentMap(filename string) (*PersistentMap, error) {
+	pm := &PersistentMap{filename, make(map[string]interface{})}
+	return pm, pm.Flush()
+}
+
+/*
+LoadPersistentMap loads a persistent map from a file.
+*/
+func LoadPersistentMap(filename string) (*PersistentMap, error) {
+	file, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0660)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	pm := &PersistentMap{filename, make(map[string]interface{})}
+
+	de := gob.NewDecoder(file)
+
+	return pm, de.Decode(&pm.Data)
+}
+
+/*
+Flush writes contents of the persistent map to the disk.
+*/
+func (pm *PersistentMap) Flush() error {
+	file, err := os.OpenFile(pm.filename, os.O_CREATE|os.O_RDWR, 0660)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	en := gob.NewEncoder(file)
+
+	return en.Encode(pm.Data)
+}
+
+/*
+PersistentStringMap is a persistent map storing string values.
+*/
+type PersistentStringMap struct {
+	filename string            // File of the persistent map
+	Data     map[string]string // Data of the persistent map
+}
+
+/*
+NewPersistentStringMap creates a new persistent map.
+*/
+func NewPersistentStringMap(filename string) (*PersistentStringMap, error) {
+	pm := &PersistentStringMap{filename, make(map[string]string)}
+	return pm, pm.Flush()
+}
+
+/*
+LoadPersistentStringMap loads a persistent map from a file.
+*/
+func LoadPersistentStringMap(filename string) (*PersistentStringMap, error) {
+	file, err := os.OpenFile(filename, os.O_CREATE|os.O_RDWR, 0660)
+	if err != nil {
+		return nil, err
+	}
+
+	pm := &PersistentStringMap{filename, make(map[string]string)}
+
+	de := gob.NewDecoder(file)
+	de.Decode(&pm.Data)
+
+	return pm, file.Close()
+}
+
+/*
+Flush writes contents of the persistent map to the disk.
+*/
+func (pm *PersistentStringMap) Flush() error {
+	file, err := os.OpenFile(pm.filename, os.O_CREATE|os.O_RDWR, 0660)
+	if err != nil {
+		return err
+	}
+
+	en := gob.NewEncoder(file)
+	en.Encode(pm.Data)
+
+	return file.Close()
+}

+ 146 - 0
datautil/persistentmap_test.go