server.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522
  1. /*
  2. * EliasDB
  3. *
  4. * Copyright 2016 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the Mozilla Public
  7. * License, v. 2.0. If a copy of the MPL was not distributed with this
  8. * file, You can obtain one at http://mozilla.org/MPL/2.0/.
  9. */
  10. /*
  11. Package server contains the code for the EliasDB server.
  12. */
  13. package server
  14. import (
  15. "bytes"
  16. "fmt"
  17. "io/ioutil"
  18. "log"
  19. "net/http"
  20. "os"
  21. "path/filepath"
  22. "sync"
  23. "time"
  24. "devt.de/krotik/common/cryptutil"
  25. "devt.de/krotik/common/datautil"
  26. "devt.de/krotik/common/errorutil"
  27. "devt.de/krotik/common/fileutil"
  28. "devt.de/krotik/common/httputil"
  29. "devt.de/krotik/common/httputil/access"
  30. "devt.de/krotik/common/httputil/auth"
  31. "devt.de/krotik/common/httputil/user"
  32. "devt.de/krotik/common/lockutil"
  33. "devt.de/krotik/common/timeutil"
  34. "devt.de/krotik/eliasdb/api"
  35. "devt.de/krotik/eliasdb/api/ac"
  36. v1 "devt.de/krotik/eliasdb/api/v1"
  37. "devt.de/krotik/eliasdb/cluster"
  38. "devt.de/krotik/eliasdb/cluster/manager"
  39. "devt.de/krotik/eliasdb/config"
  40. "devt.de/krotik/eliasdb/ecal"
  41. "devt.de/krotik/eliasdb/graph"
  42. "devt.de/krotik/eliasdb/graph/graphstorage"
  43. )
  44. /*
  45. Using custom consolelogger type so we can test log.Fatal calls with unit tests. Overwrite
  46. these if the server should not call os.Exit on a fatal error.
  47. */
  48. type consolelogger func(v ...interface{})
  49. var fatal = consolelogger(log.Fatal)
  50. var print = consolelogger(log.Print)
  51. /*
  52. Base path for all file (used by unit tests)
  53. */
  54. var basepath = ""
  55. /*
  56. UserDBPassphrase is the passphrase which will be used for the user db (only used if
  57. access control is enabled)
  58. */
  59. var UserDBPassphrase = ""
  60. /*
  61. StartServer runs the EliasDB server. The server uses config.Config for all its configuration
  62. parameters.
  63. */
  64. func StartServer() {
  65. StartServerWithSingleOp(nil)
  66. }
  67. /*
  68. StartServerWithSingleOp runs the EliasDB server. If the singleOperation function is
  69. not nil then the server executes the function and exists if the function returns true.
  70. */
  71. func StartServerWithSingleOp(singleOperation func(*graph.Manager) bool) {
  72. var err error
  73. var gs graphstorage.Storage
  74. print(fmt.Sprintf("EliasDB %v", config.ProductVersion))
  75. // Ensure we have a configuration - use the default configuration if nothing was set
  76. if config.Config == nil {
  77. config.LoadDefaultConfig()
  78. }
  79. // Create graph storage
  80. if config.Bool(config.MemoryOnlyStorage) {
  81. print("Starting memory only datastore")
  82. gs = graphstorage.NewMemoryGraphStorage(config.MemoryOnlyStorage)
  83. if config.Bool(config.EnableReadOnly) {
  84. print("Ignoring EnableReadOnly setting")
  85. }
  86. } else {
  87. loc := filepath.Join(basepath, config.Str(config.LocationDatastore))
  88. readonly := config.Bool(config.EnableReadOnly)
  89. if readonly {
  90. print("Starting datastore (readonly) in ", loc)
  91. } else {
  92. print("Starting datastore in ", loc)
  93. }
  94. // Ensure path for database exists
  95. ensurePath(loc)
  96. gs, err = graphstorage.NewDiskGraphStorage(loc, readonly)
  97. if err != nil {
  98. fatal(err)
  99. return
  100. }
  101. }
  102. // Check if clustering is enabled
  103. if config.Bool(config.EnableCluster) {
  104. print("Reading cluster config")
  105. cconfig, err := fileutil.LoadConfig(filepath.Join(basepath, config.Str(config.ClusterConfigFile)),
  106. manager.DefaultConfig)
  107. if err != nil {
  108. fatal("Failed to load cluster config:", err)
  109. return
  110. }
  111. print("Opening cluster state info")
  112. si, err := manager.NewDefaultStateInfo(filepath.Join(basepath, config.Str(config.ClusterStateInfoFile)))
  113. if err != nil {
  114. fatal("Failed to load cluster state info:", err)
  115. return
  116. }
  117. loghist := int(config.Int(config.ClusterLogHistory))
  118. print(fmt.Sprintf("Starting cluster (log history: %v)", loghist))
  119. ds, err := cluster.NewDistributedStorage(gs, cconfig, si)
  120. if err != nil {
  121. fatal("Failed to create distributed storage:", err)
  122. return
  123. }
  124. gs = ds
  125. // Make the distributed storage and the cluster log available for the REST API
  126. api.DD = ds
  127. api.DDLog = datautil.NewRingBuffer(loghist)
  128. logFunc := func(v ...interface{}) {
  129. api.DDLog.Log(timeutil.MakeTimestamp(), " ", fmt.Sprint(v...))
  130. }
  131. logPrintFunc := func(v ...interface{}) {
  132. print("[Cluster] ", fmt.Sprint(v...))
  133. api.DDLog.Log(timeutil.MakeTimestamp(), " ", fmt.Sprint(v...))
  134. }
  135. manager.LogDebug = logFunc
  136. manager.LogInfo = logPrintFunc
  137. // Kick off the cluster
  138. ds.MemberManager.Start()
  139. }
  140. // Create GraphManager
  141. print("Creating GraphManager instance")
  142. api.GS = gs
  143. api.GM = graph.NewGraphManager(gs)
  144. defer func() {
  145. print("Closing datastore")
  146. if err := gs.Close(); err != nil {
  147. fatal(err)
  148. return
  149. }
  150. os.RemoveAll(filepath.Join(basepath, config.Str(config.LockFile)))
  151. }()
  152. // Create ScriptingInterpreter instance and run ECAL scripts
  153. if config.Bool(config.EnableECALScripts) {
  154. // Make sure the script directory exists
  155. loc := filepath.Join(basepath, config.Str(config.ECALScriptFolder))
  156. ensurePath(loc)
  157. print("Loading ECAL scripts in ", loc)
  158. api.SI = ecal.NewScriptingInterpreter(loc, api.GM)
  159. if err := api.SI.Run(); err != nil {
  160. fatal("Failed to start ECAL scripting interpreter:", err)
  161. return
  162. }
  163. }
  164. // Handle single operation - these are operations which work on the GraphManager
  165. // and then exit.
  166. if singleOperation != nil && singleOperation(api.GM) {
  167. return
  168. }
  169. // Setting other API parameters
  170. // Setup cookie expiry
  171. cookieMaxAge := int(config.Int(config.CookieMaxAgeSeconds))
  172. auth.CookieMaxLifetime = cookieMaxAge
  173. user.CookieMaxLifetime = cookieMaxAge
  174. user.UserSessionManager.Provider.(*user.MemorySessionProvider).SetExpiry(cookieMaxAge)
  175. api.APIHost = config.Str(config.HTTPSHost) + ":" + config.Str(config.HTTPSPort)
  176. v1.ResultCacheMaxSize = uint64(config.Int(config.ResultCacheMaxSize))
  177. v1.ResultCacheMaxAge = config.Int(config.ResultCacheMaxAgeSeconds)
  178. // Check if HTTPS key and certificate are in place
  179. keyPath := filepath.Join(basepath, config.Str(config.LocationHTTPS), config.Str(config.HTTPSKey))
  180. certPath := filepath.Join(basepath, config.Str(config.LocationHTTPS), config.Str(config.HTTPSCertificate))
  181. keyExists, _ := fileutil.PathExists(keyPath)
  182. certExists, _ := fileutil.PathExists(certPath)
  183. if !keyExists || !certExists {
  184. // Ensure path for ssl files exists
  185. ensurePath(filepath.Join(basepath, config.Str(config.LocationHTTPS)))
  186. print("Creating key (", config.Str(config.HTTPSKey), ") and certificate (",
  187. config.Str(config.HTTPSCertificate), ") in: ", config.Str(config.LocationHTTPS))
  188. // Generate a certificate and private key
  189. err = cryptutil.GenCert(filepath.Join(basepath, config.Str(config.LocationHTTPS)),
  190. config.Str(config.HTTPSCertificate), config.Str(config.HTTPSKey),
  191. "localhost", "", 365*24*time.Hour, false, 4096, "")
  192. if err != nil {
  193. fatal("Failed to generate ssl key and certificate:", err)
  194. return
  195. }
  196. }
  197. // Register public REST endpoints - these will never be checked for authentication
  198. api.RegisterRestEndpoints(api.GeneralEndpointMap)
  199. api.RegisterRestEndpoints(v1.V1PublicEndpointMap)
  200. // Setup access control
  201. if config.Bool(config.EnableAccessControl) {
  202. // Register REST endpoints for access control
  203. api.RegisterRestEndpoints(ac.PublicAccessControlEndpointMap)
  204. // Setup user database
  205. ac.UserDB, err = datautil.NewEnforcedUserDB(filepath.Join(basepath, config.Str(config.LocationUserDB)),
  206. UserDBPassphrase)
  207. if err == nil {
  208. var ok bool
  209. // Setup access control - this will initialise the global ACL (access
  210. // control lists) object
  211. if ok, err = fileutil.PathExists(filepath.Join(basepath, config.Str(config.LocationAccessDB))); !ok && err == nil {
  212. err = ioutil.WriteFile(filepath.Join(basepath, config.Str(config.LocationAccessDB)), ac.DefaultAccessDB, 0600)
  213. }
  214. if err == nil {
  215. tab, err := access.NewPersistedACLTable(filepath.Join(basepath, config.Str(config.LocationAccessDB)), 3*time.Second)
  216. if err == nil {
  217. ac.InitACLs(tab)
  218. }
  219. }
  220. }
  221. if err == nil {
  222. // Make sure there are the initial accounts (circumventing the
  223. // enforced password constrains by using the embedded UserDB directly)
  224. if len(ac.UserDB.AllUsers()) == 0 {
  225. ac.UserDB.UserDB.AddUserEntry("elias", "elias", nil)
  226. ac.UserDB.UserDB.AddUserEntry("johndoe", "doe", nil)
  227. }
  228. // Setup the AuthHandler object which provides cookie based authentication
  229. // for endpoints which are registered with its HandleFunc
  230. ac.AuthHandler = auth.NewCookieAuthHandleFuncWrapper(http.HandleFunc)
  231. // Connect the UserDB object to the AuthHandler - this provides authentication for users
  232. ac.AuthHandler.SetAuthFunc(ac.UserDB.CheckUserPassword)
  233. // Connect the ACL object to the AuthHandler - this provides authorization for users
  234. ac.AuthHandler.SetAccessFunc(ac.ACL.CheckHTTPRequest)
  235. // Make login page a "public" page i.e. a page which can be reached without
  236. // authentication
  237. ac.AuthHandler.AddPublicPage("/login.html",
  238. httputil.SingleFileServer(filepath.Join(
  239. config.Str(config.LocationWebFolder), "login.html"),
  240. nil).ServeHTTP)
  241. // Also make the fingerprint.json a public page
  242. ac.AuthHandler.AddPublicPage("/fingerprint.json",
  243. httputil.SingleFileServer(filepath.Join(
  244. config.Str(config.LocationWebFolder), "fingerprint.json"),
  245. nil).ServeHTTP)
  246. // Adding special handlers which redirect to the login page
  247. ac.AuthHandler.CallbackSessionExpired = ac.CallbackSessionExpired
  248. ac.AuthHandler.CallbackUnauthorized = ac.CallbackUnauthorized
  249. // Finally set the HandleFunc of the AuthHandler as the HandleFunc of the API
  250. api.HandleFunc = ac.AuthHandler.HandleFunc
  251. // After the api.HandleFunc has been set we can now register the management
  252. // endpoints which should be subject to access control
  253. api.RegisterRestEndpoints(ac.AccessManagementEndpointMap)
  254. }
  255. }
  256. // Register EliasDB API endpoints - depending on if access control has been enabled
  257. // these will require authentication and authorization for a given user
  258. api.RegisterRestEndpoints(v1.V1EndpointMap)
  259. // Register normal web server
  260. if config.Bool(config.EnableWebFolder) {
  261. webFolder := filepath.Join(basepath, config.Str(config.LocationWebFolder))
  262. print("Ensuring web folder: ", webFolder)
  263. ensurePath(webFolder)
  264. fs := http.FileServer(http.Dir(webFolder))
  265. api.HandleFunc("/", fs.ServeHTTP)
  266. // Write login
  267. if config.Bool(config.EnableAccessControl) {
  268. loginFile := filepath.Join(webFolder, "login.html")
  269. print("Ensuring login page: ", loginFile)
  270. if res, _ := fileutil.PathExists(loginFile); !res {
  271. errorutil.AssertOk(ioutil.WriteFile(loginFile, []byte(LoginSRC[1:]), 0644))
  272. }
  273. }
  274. // Write terminal(s)
  275. if config.Bool(config.EnableWebTerminal) {
  276. ensurePath(filepath.Join(webFolder, api.APIRoot))
  277. termFile := filepath.Join(webFolder, api.APIRoot, "term.html")
  278. print("Ensuring web terminal: ", termFile)
  279. if res, _ := fileutil.PathExists(termFile); !res {
  280. errorutil.AssertOk(ioutil.WriteFile(termFile, []byte(TermSRC[1:]), 0644))
  281. }
  282. }
  283. if config.Bool(config.EnableClusterTerminal) {
  284. ensurePath(filepath.Join(webFolder, api.APIRoot))
  285. termFile := filepath.Join(webFolder, api.APIRoot, "cluster.html")
  286. if config.Bool(config.EnableCluster) {
  287. // Add the url to the member info of the member manager
  288. api.DD.MemberManager.MemberInfo()[manager.MemberInfoTermURL] =
  289. fmt.Sprintf("https://%v:%v%v/%v", config.Str(config.HTTPSHost),
  290. config.Str(config.HTTPSPort), api.APIRoot, "cluster.html")
  291. }
  292. print("Ensuring cluster terminal: ", termFile)
  293. if res, _ := fileutil.PathExists(termFile); !res {
  294. errorutil.AssertOk(ioutil.WriteFile(termFile, []byte(ClusterTermSRC[1:]), 0644))
  295. }
  296. }
  297. }
  298. // Start HTTPS server and enable REST API
  299. hs := &httputil.HTTPServer{}
  300. var wg sync.WaitGroup
  301. wg.Add(1)
  302. port := config.Str(config.HTTPSPort)
  303. print("Starting HTTPS server on: ", api.APIHost)
  304. go hs.RunHTTPSServer(basepath+config.Str(config.LocationHTTPS), config.Str(config.HTTPSCertificate),
  305. config.Str(config.HTTPSKey), ":"+port, &wg)
  306. // Wait until the server has started
  307. wg.Wait()
  308. // HTTPS Server has started
  309. if hs.LastError != nil {
  310. fatal(hs.LastError)
  311. return
  312. }
  313. // Add to the wait group so we can wait for the shutdown
  314. wg.Add(1)
  315. // Read server certificate and write a fingerprint file
  316. fpfile := filepath.Join(basepath, config.Str(config.LocationWebFolder), "fingerprint.json")
  317. print("Writing fingerprint file: ", fpfile)
  318. certs, _ := cryptutil.ReadX509CertsFromFile(certPath)
  319. if len(certs) > 0 {
  320. buf := bytes.Buffer{}
  321. buf.WriteString("{\n")
  322. buf.WriteString(fmt.Sprintf(` "md5" : "%s",`, cryptutil.Md5CertFingerprint(certs[0])))
  323. buf.WriteString("\n")
  324. buf.WriteString(fmt.Sprintf(` "sha1" : "%s",`, cryptutil.Sha1CertFingerprint(certs[0])))
  325. buf.WriteString("\n")
  326. buf.WriteString(fmt.Sprintf(` "sha256" : "%s"`, cryptutil.Sha256CertFingerprint(certs[0])))
  327. buf.WriteString("\n")
  328. buf.WriteString("}\n")
  329. ioutil.WriteFile(fpfile, buf.Bytes(), 0644)
  330. }
  331. // Create a lockfile so the server can be shut down
  332. lf := lockutil.NewLockFile(basepath+config.Str(config.LockFile), time.Duration(2)*time.Second)
  333. lf.Start()
  334. go func() {
  335. // Check if the lockfile watcher is running and
  336. // call shutdown once it has finished
  337. for lf.WatcherRunning() {
  338. time.Sleep(time.Duration(1) * time.Second)
  339. }
  340. print("Lockfile was modified")
  341. hs.Shutdown()
  342. }()
  343. print("Waiting for shutdown")
  344. wg.Wait()
  345. print("Shutting down")
  346. if config.Bool(config.EnableCluster) {
  347. // Shutdown cluster
  348. gs.(*cluster.DistributedStorage).MemberManager.Shutdown()
  349. }
  350. }
  351. /*
  352. ensurePath ensures that a given relative path exists.
  353. */
  354. func ensurePath(path string) {
  355. if res, _ := fileutil.PathExists(path); !res {
  356. if err := os.Mkdir(path, 0770); err != nil {
  357. fatal("Could not create directory:", err.Error())
  358. return
  359. }
  360. }
  361. }