branch.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766
  1. /*
  2. * Rufs - Remote Union File System
  3. *
  4. * Copyright 2017 Matthias Ladkau. All rights reserved.
  5. *
  6. * This Source Code Form is subject to the terms of the MIT
  7. * License, If a copy of the MIT License was not distributed with this
  8. * file, You can obtain one at https://opensource.org/licenses/MIT.
  9. */
  10. /*
  11. Package rufs contains the main API to Rufs.
  12. Rufs is organized as a collection of branches. Each branch represents a physical
  13. file system structure which can be queried and updated by an authorized client.
  14. On the client side one or several branches are organized into a tree. The
  15. single branches can overlay each other. For example:
  16. Branch A
  17. /foo/A
  18. /foo/B
  19. /bar/C
  20. Branch B
  21. /foo/C
  22. /test/D
  23. Tree 1
  24. /myspace => Branch A, Branch B
  25. Accessing tree with:
  26. /myspace/foo/A gets file /foo/A from Branch A while
  27. /myspace/foo/C gets file /foo/C from Branch B
  28. Write operations go only to branches which are mapped as writing branches
  29. and who accept them (i.e. are not set to readonly on the side of the branch).
  30. */
  31. package rufs
  32. import (
  33. "bytes"
  34. "crypto/tls"
  35. "encoding/gob"
  36. "fmt"
  37. "io"
  38. "io/ioutil"
  39. "os"
  40. "path"
  41. "path/filepath"
  42. "regexp"
  43. "strconv"
  44. "strings"
  45. "devt.de/krotik/common/errorutil"
  46. "devt.de/krotik/common/fileutil"
  47. "devt.de/krotik/common/pools"
  48. "devt.de/krotik/common/stringutil"
  49. "devt.de/krotik/rufs/config"
  50. "devt.de/krotik/rufs/node"
  51. )
  52. func init() {
  53. // Make sure we can use the relevant types in a gob operation
  54. gob.Register([][]os.FileInfo{})
  55. gob.Register(&FileInfo{})
  56. }
  57. /*
  58. Branch models a single exported branch in Rufs.
  59. */
  60. type Branch struct {
  61. rootPath string // Local directory (absolute path) modeling the branch root
  62. node *node.RufsNode // Local RPC node
  63. readonly bool // Flag if this branch is readonly
  64. }
  65. /*
  66. NewBranch returns a new exported branch.
  67. */
  68. func NewBranch(cfg map[string]interface{}, cert *tls.Certificate) (*Branch, error) {
  69. var err error
  70. var b *Branch
  71. // Make sure the given config is ok
  72. if err = config.CheckBranchExportConfig(cfg); err == nil {
  73. // Create RPC server
  74. addr := fmt.Sprintf("%v:%v", fileutil.ConfStr(cfg, config.RPCHost),
  75. fileutil.ConfStr(cfg, config.RPCPort))
  76. rn := node.NewNode(addr, fileutil.ConfStr(cfg, config.BranchName),
  77. fileutil.ConfStr(cfg, config.BranchSecret), cert, nil)
  78. // Start the rpc server
  79. if err = rn.Start(cert); err == nil {
  80. var rootPath string
  81. // Construct root path
  82. if rootPath, err = filepath.Abs(fileutil.ConfStr(cfg, config.LocalFolder)); err == nil {
  83. b = &Branch{rootPath, rn, fileutil.ConfBool(cfg, config.EnableReadOnly)}
  84. rn.DataHandler = b.requestHandler
  85. }
  86. }
  87. }
  88. return b, err
  89. }
  90. /*
  91. Name returns the name of the branch.
  92. */
  93. func (b *Branch) Name() string {
  94. return b.node.Name()
  95. }
  96. /*
  97. SSLFingerprint returns the SSL fingerprint of the branch.
  98. */
  99. func (b *Branch) SSLFingerprint() string {
  100. return b.node.SSLFingerprint()
  101. }
  102. /*
  103. Shutdown shuts the branch down.
  104. */
  105. func (b *Branch) Shutdown() error {
  106. return b.node.Shutdown()
  107. }
  108. /*
  109. IsReadOnly returns if this branch is read-only.
  110. */
  111. func (b *Branch) IsReadOnly() bool {
  112. return b.readonly
  113. }
  114. /*
  115. checkReadOnly returns an error if this branch is read-only.
  116. */
  117. func (b *Branch) checkReadOnly() error {
  118. var err error
  119. if b.IsReadOnly() {
  120. err = fmt.Errorf("Branch %v is read-only", b.Name())
  121. }
  122. return err
  123. }
  124. // Branch API
  125. // ==========
  126. /*
  127. Dir returns file listings matching a given pattern of one or more directories.
  128. The contents of the given path is returned along with checksums if the checksum
  129. flag is specified. Optionally, also the contents of all subdirectories can be
  130. returned if the recursive flag is set. The return values is a list of traversed
  131. directories (platform-agnostic) and their corresponding contents.
  132. */
  133. func (b *Branch) Dir(spath string, pattern string, recursive bool, checksums bool) ([]string, [][]os.FileInfo, error) {
  134. var fis []os.FileInfo
  135. // Compile pattern
  136. re, err := regexp.Compile(pattern)
  137. if err != nil {
  138. return nil, nil, err
  139. }
  140. createRufsFileInfos := func(dirname string, afis []os.FileInfo) []os.FileInfo {
  141. var fis []os.FileInfo
  142. fis = make([]os.FileInfo, 0, len(afis))
  143. for _, fi := range afis {
  144. // Append if it matches the pattern
  145. if re.MatchString(fi.Name()) {
  146. fis = append(fis, fi)
  147. }
  148. }
  149. // Wrap normal file infos and calculate checksum if necessary
  150. ret := WrapFileInfos(dirname, fis)
  151. if checksums {
  152. for _, fi := range fis {
  153. if !fi.IsDir() {
  154. // The sum is either there or not ... - access errors should
  155. // be caught when trying to read the file
  156. sum, _ := fileutil.CheckSumFileFast(filepath.Join(dirname, fi.Name()))
  157. fi.(*FileInfo).FiChecksum = sum
  158. }
  159. }
  160. }
  161. return ret
  162. }
  163. subPath, err := b.constructSubPath(spath)
  164. if err == nil {
  165. if !recursive {
  166. if fis, err = ioutil.ReadDir(subPath); err == nil {
  167. return []string{spath},
  168. [][]os.FileInfo{createRufsFileInfos(subPath, fis)}, nil
  169. }
  170. } else {
  171. var rpaths []string
  172. var rfis [][]os.FileInfo
  173. var addSubDir func(string, string) error
  174. // Recursive function to walk directories and symlinks
  175. // in a platform-agnostic way
  176. addSubDir = func(p string, rp string) error {
  177. fis, err = ioutil.ReadDir(p)
  178. if err == nil {
  179. rpaths = append(rpaths, rp)
  180. rfis = append(rfis, createRufsFileInfos(p, fis))
  181. for _, fi := range fis {
  182. if err == nil && fi.IsDir() {
  183. err = addSubDir(filepath.Join(p, fi.Name()),
  184. path.Join(rp, fi.Name()))
  185. }
  186. }
  187. }
  188. return err
  189. }
  190. if err = addSubDir(subPath, spath); err == nil {
  191. return rpaths, rfis, nil
  192. }
  193. }
  194. }
  195. // Ignore any not exists errors
  196. if os.IsNotExist(err) {
  197. err = nil
  198. }
  199. return nil, nil, err
  200. }
  201. /*
  202. ReadFileToBuffer reads a complete file into a given buffer which implements
  203. io.Writer.
  204. */
  205. func (b *Branch) ReadFileToBuffer(spath string, buf io.Writer) error {
  206. var n int
  207. var err error
  208. var offset int64
  209. readBuf := make([]byte, DefaultReadBufferSize)
  210. for err == nil {
  211. n, err = b.ReadFile(spath, readBuf, offset)
  212. if err == nil {
  213. _, err = buf.Write(readBuf[:n])
  214. offset += int64(n)
  215. } else if IsEOF(err) {
  216. // We reached the end of the file
  217. err = nil
  218. break
  219. }
  220. }
  221. return err
  222. }
  223. /*
  224. ReadFile reads up to len(p) bytes into p from the given offset. It
  225. returns the number of bytes read (0 <= n <= len(p)) and any error
  226. encountered.
  227. */
  228. func (b *Branch) ReadFile(spath string, p []byte, offset int64) (int, error) {
  229. var n int
  230. subPath, err := b.constructSubPath(spath)
  231. if err == nil {
  232. var fi os.FileInfo
  233. if fi, err = os.Stat(subPath); err == nil {
  234. if fi.IsDir() {
  235. err = fmt.Errorf("read /%v: is a directory", spath)
  236. } else if err == nil {
  237. var f *os.File
  238. if f, err = os.Open(subPath); err == nil {
  239. defer f.Close()
  240. sr := io.NewSectionReader(f, 0, fi.Size())
  241. if _, err = sr.Seek(offset, io.SeekStart); err == nil {
  242. n, err = sr.Read(p)
  243. }
  244. }
  245. }
  246. }
  247. }
  248. return n, err
  249. }
  250. /*
  251. WriteFileFromBuffer writes a complete file from a given buffer which implements
  252. io.Reader.
  253. */
  254. func (b *Branch) WriteFileFromBuffer(spath string, buf io.Reader) error {
  255. var err error
  256. var offset int64
  257. if err = b.checkReadOnly(); err == nil {
  258. writeBuf := make([]byte, DefaultReadBufferSize)
  259. for err == nil {
  260. var n int
  261. if n, err = buf.Read(writeBuf); err == nil {
  262. _, err = b.WriteFile(spath, writeBuf[:n], offset)
  263. offset += int64(n)
  264. } else if IsEOF(err) {
  265. // We reached the end of the file
  266. b.WriteFile(spath, []byte{}, offset)
  267. err = nil
  268. break
  269. }
  270. }
  271. }
  272. return err
  273. }
  274. /*
  275. WriteFile writes p into the given file from the given offset. It
  276. returns the number of written bytes and any error encountered.
  277. */
  278. func (b *Branch) WriteFile(spath string, p []byte, offset int64) (int, error) {
  279. var n int
  280. var m int64
  281. if err := b.checkReadOnly(); err != nil {
  282. return 0, err
  283. }
  284. buf := byteSlicePool.Get().([]byte)
  285. defer func() {
  286. byteSlicePool.Put(buf)
  287. }()
  288. growFile := func(f *os.File, n int64) {
  289. var err error
  290. toWrite := n
  291. for err == nil && toWrite > 0 {
  292. if toWrite > int64(DefaultReadBufferSize) {
  293. _, err = f.Write(buf[:DefaultReadBufferSize])
  294. toWrite -= int64(DefaultReadBufferSize)
  295. } else {
  296. _, err = f.Write(buf[:toWrite])
  297. toWrite = 0
  298. }
  299. }
  300. }
  301. subPath, err := b.constructSubPath(spath)
  302. if err == nil {
  303. var fi os.FileInfo
  304. var f *os.File
  305. if fi, err = os.Stat(subPath); os.IsNotExist(err) {
  306. // Ensure path exists
  307. dir, _ := filepath.Split(subPath)
  308. if err = os.MkdirAll(dir, 0755); err == nil {
  309. // Create the file newly
  310. if f, err = os.OpenFile(subPath, os.O_RDWR|os.O_CREATE, 0644); err == nil {
  311. defer f.Close()
  312. if offset > 0 {
  313. growFile(f, offset)
  314. }
  315. m, err = io.Copy(f, bytes.NewBuffer(p))
  316. n += int(m)
  317. }
  318. }
  319. } else if err == nil {
  320. // File does exist
  321. if f, err := os.OpenFile(subPath, os.O_RDWR, 0644); err == nil {
  322. defer f.Close()
  323. if fi.Size() < offset {
  324. f.Seek(fi.Size(), io.SeekStart)
  325. growFile(f, offset-fi.Size())
  326. } else {
  327. f.Seek(offset, io.SeekStart)
  328. }
  329. m, err = io.Copy(f, bytes.NewBuffer(p))
  330. errorutil.AssertOk(err)
  331. n += int(m)
  332. }
  333. }
  334. }
  335. return n, err
  336. }
  337. /*
  338. ItemOp parameter
  339. */
  340. const (
  341. ItemOpAction = "itemop_action" // ItemOp action
  342. ItemOpName = "itemop_name" // Item name
  343. ItemOpNewName = "itemop_newname" // New item name
  344. )
  345. /*
  346. ItemOp actions
  347. */
  348. const (
  349. ItemOpActRename = "rename" // Rename a file or directory
  350. ItemOpActDelete = "delete" // Delete a file or directory
  351. ItemOpActMkDir = "mkdir" // Create a directory
  352. )
  353. /*
  354. ItemOp executes a file or directory specific operation which can either
  355. succeed or fail (e.g. rename or delete). Actions and parameters should
  356. be given in the opdata map.
  357. */
  358. func (b *Branch) ItemOp(spath string, opdata map[string]string) (bool, error) {
  359. res := false
  360. if err := b.checkReadOnly(); err != nil {
  361. return false, err
  362. }
  363. subPath, err := b.constructSubPath(spath)
  364. if err == nil {
  365. action := opdata[ItemOpAction]
  366. fileFromOpData := func(key string) (string, error) {
  367. // Make sure we are only dealing with files
  368. _, name := filepath.Split(opdata[key])
  369. if name == "" {
  370. return "", fmt.Errorf("This operation requires a specific file or directory")
  371. }
  372. // Build the relative paths
  373. return filepath.Join(filepath.FromSlash(subPath), name), nil
  374. }
  375. if action == ItemOpActMkDir {
  376. var name string
  377. // Make directory action
  378. if name, err = fileFromOpData(ItemOpName); err == nil {
  379. err = os.MkdirAll(name, 0755)
  380. }
  381. } else if action == ItemOpActRename {
  382. var name, newname string
  383. // Rename action
  384. if name, err = fileFromOpData(ItemOpName); err == nil {
  385. if newname, err = fileFromOpData(ItemOpNewName); err == nil {
  386. err = os.Rename(name, newname)
  387. }
  388. }
  389. } else if action == ItemOpActDelete {
  390. var name string
  391. // Delete action
  392. if name, err = fileFromOpData(ItemOpName); err == nil {
  393. del := func(name string) error {
  394. var err error
  395. if ok, _ := fileutil.PathExists(name); ok {
  396. err = os.RemoveAll(name)
  397. } else {
  398. err = os.ErrNotExist
  399. }
  400. return err
  401. }
  402. if strings.Contains(name, "*") {
  403. var rex string
  404. // We have a wildcard
  405. rootdir, glob := filepath.Split(name)
  406. // Create a regex from the given glob expression
  407. if rex, err = stringutil.GlobToRegex(glob); err == nil {
  408. var dirs []string
  409. var fis [][]os.FileInfo
  410. if dirs, fis, err = b.Dir(spath, rex, true, false); err == nil {
  411. for i, dir := range dirs {
  412. // Remove all files and dirs according to the wildcard
  413. for _, fi := range fis[i] {
  414. os.RemoveAll(filepath.Join(rootdir,
  415. filepath.FromSlash(dir), fi.Name()))
  416. }
  417. }
  418. }
  419. }
  420. } else {
  421. err = del(name)
  422. }
  423. }
  424. }
  425. // Determine if we succeeded
  426. res = err == nil || os.IsNotExist(err)
  427. }
  428. return res, err
  429. }
  430. // Request handling functions
  431. // ==========================
  432. /*
  433. DefaultReadBufferSize is the default size for file reading.
  434. */
  435. var DefaultReadBufferSize = 1024 * 16
  436. /*
  437. bufferPool holds buffers which are used to marshal objects.
  438. */
  439. var bufferPool = pools.NewByteBufferPool()
  440. /*
  441. byteSlicePool holds buffers which are used to read files
  442. */
  443. var byteSlicePool = pools.NewByteSlicePool(DefaultReadBufferSize)
  444. /*
  445. Meta parameter
  446. */
  447. const (
  448. ParamAction = "a" // Requested action
  449. ParamPath = "p" // Path string
  450. ParamPattern = "x" // Pattern string
  451. ParamRecursive = "r" // Recursive flag
  452. ParamChecksums = "c" // Checksum flag
  453. ParamOffset = "o" // Offset parameter
  454. ParamSize = "s" // Size parameter
  455. )
  456. /*
  457. Possible actions
  458. */
  459. const (
  460. OpDir = "dir" // Read the contents of a path
  461. OpRead = "read" // Read the contents of a file
  462. OpWrite = "write" // Read the contents of a file
  463. OpItemOp = "itemop" // File or directory operation
  464. )
  465. /*
  466. requestHandler handles incoming requests from other branches or trees.
  467. */
  468. func (b *Branch) requestHandler(ctrl map[string]string, data []byte) ([]byte, error) {
  469. var err error
  470. var res interface{}
  471. var ret []byte
  472. action := ctrl[ParamAction]
  473. // Handle operation requests
  474. if action == OpDir {
  475. var dirs []string
  476. var fis [][]os.FileInfo
  477. dir := ctrl[ParamPath]
  478. pattern := ctrl[ParamPattern]
  479. rec := strings.ToLower(ctrl[ParamRecursive]) == "true"
  480. sum := strings.ToLower(ctrl[ParamChecksums]) == "true"
  481. if dirs, fis, err = b.Dir(dir, pattern, rec, sum); err == nil {
  482. res = []interface{}{dirs, fis}
  483. }
  484. } else if action == OpItemOp {
  485. res, err = b.ItemOp(ctrl[ParamPath], ctrl)
  486. } else if action == OpRead {
  487. var size, n int
  488. var offset int64
  489. spath := ctrl[ParamPath]
  490. if size, err = strconv.Atoi(ctrl[ParamSize]); err == nil {
  491. if offset, err = strconv.ParseInt(ctrl[ParamOffset], 10, 64); err == nil {
  492. buf := byteSlicePool.Get().([]byte)
  493. defer func() {
  494. byteSlicePool.Put(buf)
  495. }()
  496. if len(buf) < size {
  497. // Constantly requesting bigger buffers will
  498. // eventually replace all default sized buffers
  499. buf = make([]byte, size)
  500. }
  501. if n, err = b.ReadFile(spath, buf[:size], offset); err == nil {
  502. res = []interface{}{n, buf[:size]}
  503. }
  504. }
  505. }
  506. } else if action == OpWrite {
  507. var offset int64
  508. spath := ctrl[ParamPath]
  509. if offset, err = strconv.ParseInt(ctrl[ParamOffset], 10, 64); err == nil {
  510. res, err = b.WriteFile(spath, data, offset)
  511. }
  512. }
  513. // Send the response
  514. if err == nil {
  515. // Allocate a new encoding buffer - no need to lock as
  516. // it is based on sync.Pool
  517. // Pooled encoding buffers are used to keep expensive buffer
  518. // reallocations to a minimum. It is better to allocate the
  519. // actual response buffer once the response size is known.
  520. bb := bufferPool.Get().(*bytes.Buffer)
  521. if err = gob.NewEncoder(bb).Encode(res); err == nil {
  522. toSend := bb.Bytes()
  523. // Allocate the response array
  524. ret = make([]byte, len(toSend))
  525. // Copy encoded result into the response array
  526. copy(ret, toSend)
  527. }
  528. // Return the encoding buffer back to the pool
  529. go func() {
  530. bb.Reset()
  531. bufferPool.Put(bb)
  532. }()
  533. }
  534. if err != nil {
  535. // Ensure we don't leak local paths - this might not work in
  536. // all situations and depends on the underlying os. In this
  537. // error messages might include information on the full local
  538. // path in error messages.
  539. absRoot, _ := filepath.Abs(b.rootPath)
  540. err = fmt.Errorf("%v", strings.Replace(err.Error(), absRoot, "", -1))
  541. }
  542. return ret, err
  543. }
  544. // Util functions
  545. // ==============
  546. func (b *Branch) constructSubPath(rpath string) (string, error) {
  547. // Produce the actual subpath - this should also produce windows
  548. // paths correctly (i.e. foo/bar -> C:\root\foo\bar)
  549. subPath := filepath.Join(b.rootPath, filepath.FromSlash(rpath))
  550. // Check that the new sub path is under the root path
  551. absSubPath, err := filepath.Abs(subPath)
  552. if err == nil {
  553. if strings.HasPrefix(absSubPath, b.rootPath) {
  554. return subPath, nil
  555. }
  556. err = fmt.Errorf("Requested path %v is outside of the branch", rpath)
  557. }
  558. return "", err
  559. }