fakefs.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953
  1. // Copyright (C) 2018 The Syncthing Authors.
  2. //
  3. // This Source Code Form is subject to the terms of the Mozilla Public
  4. // License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5. // You can obtain one at https://mozilla.org/MPL/2.0/.
  6. package fs
  7. import (
  8. "context"
  9. "errors"
  10. "fmt"
  11. "hash/fnv"
  12. "io"
  13. "math/rand"
  14. "net/url"
  15. "os"
  16. "path/filepath"
  17. "strconv"
  18. "strings"
  19. "sync"
  20. "testing"
  21. "time"
  22. )
  23. // see readShortAt()
  24. const randomBlockShift = 14 // 128k
  25. // fakeFS is a fake filesystem for testing and benchmarking. It has the
  26. // following properties:
  27. //
  28. // - File metadata is kept in RAM. Specifically, we remember which files and
  29. // directories exist, their dates, permissions and sizes. Symlinks are
  30. // not supported.
  31. //
  32. // - File contents are generated pseudorandomly with just the file name as
  33. // seed. Writes are discarded, other than having the effect of increasing
  34. // the file size. If you only write data that you've read from a file with
  35. // the same name on a different fakeFS, you'll never know the difference...
  36. //
  37. // - We totally ignore permissions - pretend you are root.
  38. //
  39. // - The root path can contain URL query-style parameters that pre populate
  40. // the filesystem at creation with a certain amount of random data:
  41. //
  42. // files=n to generate n random files (default 0)
  43. // maxsize=n to generate files up to a total of n MiB (default 0)
  44. // sizeavg=n to set the average size of random files, in bytes (default 1<<20)
  45. // seed=n to set the initial random seed (default 0)
  46. // insens=b "true" makes filesystem case-insensitive Windows- or OSX-style (default false)
  47. // latency=d to set the amount of time each "disk" operation takes, where d is time.ParseDuration format
  48. //
  49. // - Two fakeFS:s pointing at the same root path see the same files.
  50. //
  51. type fakeFS struct {
  52. counters fakeFSCounters
  53. uri string
  54. mut sync.Mutex
  55. root *fakeEntry
  56. insens bool
  57. withContent bool
  58. latency time.Duration
  59. }
  60. type fakeFSCounters struct {
  61. Chmod int64
  62. Lchown int64
  63. Chtimes int64
  64. Create int64
  65. DirNames int64
  66. Lstat int64
  67. Mkdir int64
  68. MkdirAll int64
  69. Open int64
  70. OpenFile int64
  71. ReadSymlink int64
  72. Remove int64
  73. RemoveAll int64
  74. Rename int64
  75. }
  76. var (
  77. fakeFSMut sync.Mutex
  78. fakeFSCache = make(map[string]*fakeFS)
  79. )
  80. func newFakeFilesystem(rootURI string, _ ...Option) *fakeFS {
  81. fakeFSMut.Lock()
  82. defer fakeFSMut.Unlock()
  83. var params url.Values
  84. uri, err := url.Parse(rootURI)
  85. if err == nil {
  86. params = uri.Query()
  87. }
  88. if fs, ok := fakeFSCache[rootURI]; ok {
  89. // Already have an fs at this path
  90. return fs
  91. }
  92. fs := &fakeFS{
  93. uri: "fake://" + rootURI,
  94. root: &fakeEntry{
  95. name: "/",
  96. entryType: fakeEntryTypeDir,
  97. mode: 0700,
  98. mtime: time.Now(),
  99. children: make(map[string]*fakeEntry),
  100. },
  101. }
  102. files, _ := strconv.Atoi(params.Get("files"))
  103. maxsize, _ := strconv.Atoi(params.Get("maxsize"))
  104. sizeavg, _ := strconv.Atoi(params.Get("sizeavg"))
  105. seed, _ := strconv.Atoi(params.Get("seed"))
  106. fs.insens = params.Get("insens") == "true"
  107. fs.withContent = params.Get("content") == "true"
  108. if sizeavg == 0 {
  109. sizeavg = 1 << 20
  110. }
  111. if files > 0 || maxsize > 0 {
  112. // Generate initial data according to specs. Operations in here
  113. // *look* like file I/O, but they are not. Do not worry that they
  114. // might fail.
  115. rng := rand.New(rand.NewSource(int64(seed)))
  116. var createdFiles int
  117. var writtenData int64
  118. for (files == 0 || createdFiles < files) && (maxsize == 0 || writtenData>>20 < int64(maxsize)) {
  119. dir := filepath.Join(fmt.Sprintf("%02x", rng.Intn(255)), fmt.Sprintf("%02x", rng.Intn(255)))
  120. file := fmt.Sprintf("%016x", rng.Int63())
  121. fs.MkdirAll(dir, 0755)
  122. fd, _ := fs.Create(filepath.Join(dir, file))
  123. createdFiles++
  124. fsize := int64(sizeavg/2 + rng.Intn(sizeavg))
  125. fd.Truncate(fsize)
  126. writtenData += fsize
  127. ftime := time.Unix(1000000000+rng.Int63n(10*365*86400), 0)
  128. fs.Chtimes(filepath.Join(dir, file), ftime, ftime)
  129. }
  130. }
  131. // Also create a default folder marker for good measure
  132. fs.Mkdir(".stfolder", 0700)
  133. // We only set the latency after doing the operations required to create
  134. // the filesystem initially.
  135. fs.latency, _ = time.ParseDuration(params.Get("latency"))
  136. fakeFSCache[rootURI] = fs
  137. return fs
  138. }
  139. type fakeEntryType int
  140. const (
  141. fakeEntryTypeFile fakeEntryType = iota
  142. fakeEntryTypeDir
  143. fakeEntryTypeSymlink
  144. )
  145. // fakeEntry is an entry (file or directory) in the fake filesystem
  146. type fakeEntry struct {
  147. name string
  148. entryType fakeEntryType
  149. dest string // for symlinks
  150. size int64
  151. mode FileMode
  152. uid int
  153. gid int
  154. mtime time.Time
  155. children map[string]*fakeEntry
  156. content []byte
  157. }
  158. func (fs *fakeFS) entryForName(name string) *fakeEntry {
  159. // bug: lookup doesn't work through symlinks.
  160. if fs.insens {
  161. name = UnicodeLowercaseNormalized(name)
  162. }
  163. name = filepath.ToSlash(name)
  164. if name == "." || name == "/" {
  165. return fs.root
  166. }
  167. name = strings.Trim(name, "/")
  168. comps := strings.Split(name, "/")
  169. entry := fs.root
  170. for _, comp := range comps {
  171. if entry.entryType != fakeEntryTypeDir {
  172. return nil
  173. }
  174. var ok bool
  175. entry, ok = entry.children[comp]
  176. if !ok {
  177. return nil
  178. }
  179. }
  180. return entry
  181. }
  182. func (fs *fakeFS) Chmod(name string, mode FileMode) error {
  183. fs.mut.Lock()
  184. defer fs.mut.Unlock()
  185. fs.counters.Chmod++
  186. time.Sleep(fs.latency)
  187. entry := fs.entryForName(name)
  188. if entry == nil {
  189. return os.ErrNotExist
  190. }
  191. entry.mode = mode
  192. return nil
  193. }
  194. func (fs *fakeFS) Lchown(name string, uid, gid int) error {
  195. fs.mut.Lock()
  196. defer fs.mut.Unlock()
  197. fs.counters.Lchown++
  198. time.Sleep(fs.latency)
  199. entry := fs.entryForName(name)
  200. if entry == nil {
  201. return os.ErrNotExist
  202. }
  203. entry.uid = uid
  204. entry.gid = gid
  205. return nil
  206. }
  207. func (fs *fakeFS) Chtimes(name string, atime time.Time, mtime time.Time) error {
  208. fs.mut.Lock()
  209. defer fs.mut.Unlock()
  210. fs.counters.Chtimes++
  211. time.Sleep(fs.latency)
  212. entry := fs.entryForName(name)
  213. if entry == nil {
  214. return os.ErrNotExist
  215. }
  216. entry.mtime = mtime
  217. return nil
  218. }
  219. func (fs *fakeFS) create(name string) (*fakeEntry, error) {
  220. fs.mut.Lock()
  221. defer fs.mut.Unlock()
  222. fs.counters.Create++
  223. time.Sleep(fs.latency)
  224. if entry := fs.entryForName(name); entry != nil {
  225. if entry.entryType == fakeEntryTypeDir {
  226. return nil, os.ErrExist
  227. } else if entry.entryType == fakeEntryTypeSymlink {
  228. return nil, errors.New("following symlink not supported")
  229. }
  230. entry.size = 0
  231. entry.mtime = time.Now()
  232. entry.mode = 0666
  233. entry.content = nil
  234. if fs.withContent {
  235. entry.content = make([]byte, 0)
  236. }
  237. return entry, nil
  238. }
  239. dir := filepath.Dir(name)
  240. base := filepath.Base(name)
  241. entry := fs.entryForName(dir)
  242. if entry == nil {
  243. return nil, os.ErrNotExist
  244. }
  245. new := &fakeEntry{
  246. name: base,
  247. mode: 0666,
  248. mtime: time.Now(),
  249. }
  250. if fs.insens {
  251. base = UnicodeLowercaseNormalized(base)
  252. }
  253. if fs.withContent {
  254. new.content = make([]byte, 0)
  255. }
  256. entry.children[base] = new
  257. return new, nil
  258. }
  259. func (fs *fakeFS) Create(name string) (File, error) {
  260. entry, err := fs.create(name)
  261. if err != nil {
  262. return nil, err
  263. }
  264. if fs.insens {
  265. return &fakeFile{fakeEntry: entry, presentedName: filepath.Base(name)}, nil
  266. }
  267. return &fakeFile{fakeEntry: entry}, nil
  268. }
  269. func (fs *fakeFS) CreateSymlink(target, name string) error {
  270. entry, err := fs.create(name)
  271. if err != nil {
  272. return err
  273. }
  274. entry.entryType = fakeEntryTypeSymlink
  275. entry.dest = target
  276. return nil
  277. }
  278. func (fs *fakeFS) DirNames(name string) ([]string, error) {
  279. fs.mut.Lock()
  280. defer fs.mut.Unlock()
  281. fs.counters.DirNames++
  282. time.Sleep(fs.latency)
  283. entry := fs.entryForName(name)
  284. if entry == nil {
  285. return nil, os.ErrNotExist
  286. }
  287. names := make([]string, 0, len(entry.children))
  288. for _, child := range entry.children {
  289. names = append(names, child.name)
  290. }
  291. return names, nil
  292. }
  293. func (fs *fakeFS) Lstat(name string) (FileInfo, error) {
  294. fs.mut.Lock()
  295. defer fs.mut.Unlock()
  296. fs.counters.Lstat++
  297. time.Sleep(fs.latency)
  298. entry := fs.entryForName(name)
  299. if entry == nil {
  300. return nil, os.ErrNotExist
  301. }
  302. info := &fakeFileInfo{*entry}
  303. if fs.insens {
  304. info.name = filepath.Base(name)
  305. }
  306. return info, nil
  307. }
  308. func (fs *fakeFS) Mkdir(name string, perm FileMode) error {
  309. fs.mut.Lock()
  310. defer fs.mut.Unlock()
  311. fs.counters.Mkdir++
  312. time.Sleep(fs.latency)
  313. dir := filepath.Dir(name)
  314. base := filepath.Base(name)
  315. entry := fs.entryForName(dir)
  316. key := base
  317. if entry == nil {
  318. return os.ErrNotExist
  319. }
  320. if entry.entryType != fakeEntryTypeDir {
  321. return os.ErrExist
  322. }
  323. if fs.insens {
  324. key = UnicodeLowercaseNormalized(key)
  325. }
  326. if _, ok := entry.children[key]; ok {
  327. return os.ErrExist
  328. }
  329. entry.children[key] = &fakeEntry{
  330. name: base,
  331. entryType: fakeEntryTypeDir,
  332. mode: perm,
  333. mtime: time.Now(),
  334. children: make(map[string]*fakeEntry),
  335. }
  336. return nil
  337. }
  338. func (fs *fakeFS) MkdirAll(name string, perm FileMode) error {
  339. fs.mut.Lock()
  340. defer fs.mut.Unlock()
  341. fs.counters.MkdirAll++
  342. time.Sleep(fs.latency)
  343. name = filepath.ToSlash(name)
  344. name = strings.Trim(name, "/")
  345. comps := strings.Split(name, "/")
  346. entry := fs.root
  347. for _, comp := range comps {
  348. key := comp
  349. if fs.insens {
  350. key = UnicodeLowercaseNormalized(key)
  351. }
  352. next, ok := entry.children[key]
  353. if !ok {
  354. new := &fakeEntry{
  355. name: comp,
  356. entryType: fakeEntryTypeDir,
  357. mode: perm,
  358. mtime: time.Now(),
  359. children: make(map[string]*fakeEntry),
  360. }
  361. entry.children[key] = new
  362. next = new
  363. } else if next.entryType != fakeEntryTypeDir {
  364. return errors.New("not a directory")
  365. }
  366. entry = next
  367. }
  368. return nil
  369. }
  370. func (fs *fakeFS) Open(name string) (File, error) {
  371. fs.mut.Lock()
  372. defer fs.mut.Unlock()
  373. fs.counters.Open++
  374. time.Sleep(fs.latency)
  375. entry := fs.entryForName(name)
  376. if entry == nil || entry.entryType != fakeEntryTypeFile {
  377. return nil, os.ErrNotExist
  378. }
  379. if fs.insens {
  380. return &fakeFile{fakeEntry: entry, presentedName: filepath.Base(name)}, nil
  381. }
  382. return &fakeFile{fakeEntry: entry}, nil
  383. }
  384. func (fs *fakeFS) OpenFile(name string, flags int, mode FileMode) (File, error) {
  385. if flags&os.O_CREATE == 0 {
  386. return fs.Open(name)
  387. }
  388. fs.mut.Lock()
  389. defer fs.mut.Unlock()
  390. fs.counters.OpenFile++
  391. time.Sleep(fs.latency)
  392. dir := filepath.Dir(name)
  393. base := filepath.Base(name)
  394. entry := fs.entryForName(dir)
  395. key := base
  396. if entry == nil {
  397. return nil, os.ErrNotExist
  398. } else if entry.entryType != fakeEntryTypeDir {
  399. return nil, errors.New("not a directory")
  400. }
  401. if fs.insens {
  402. key = UnicodeLowercaseNormalized(key)
  403. }
  404. if flags&os.O_EXCL != 0 {
  405. if _, ok := entry.children[key]; ok {
  406. return nil, os.ErrExist
  407. }
  408. }
  409. newEntry := &fakeEntry{
  410. name: base,
  411. mode: mode,
  412. mtime: time.Now(),
  413. }
  414. if fs.withContent {
  415. newEntry.content = make([]byte, 0)
  416. }
  417. entry.children[key] = newEntry
  418. return &fakeFile{fakeEntry: newEntry}, nil
  419. }
  420. func (fs *fakeFS) ReadSymlink(name string) (string, error) {
  421. fs.mut.Lock()
  422. defer fs.mut.Unlock()
  423. fs.counters.ReadSymlink++
  424. time.Sleep(fs.latency)
  425. entry := fs.entryForName(name)
  426. if entry == nil {
  427. return "", os.ErrNotExist
  428. } else if entry.entryType != fakeEntryTypeSymlink {
  429. return "", errors.New("not a symlink")
  430. }
  431. return entry.dest, nil
  432. }
  433. func (fs *fakeFS) Remove(name string) error {
  434. fs.mut.Lock()
  435. defer fs.mut.Unlock()
  436. fs.counters.Remove++
  437. time.Sleep(fs.latency)
  438. if fs.insens {
  439. name = UnicodeLowercaseNormalized(name)
  440. }
  441. entry := fs.entryForName(name)
  442. if entry == nil {
  443. return os.ErrNotExist
  444. }
  445. if len(entry.children) != 0 {
  446. return errors.New("not empty")
  447. }
  448. entry = fs.entryForName(filepath.Dir(name))
  449. delete(entry.children, filepath.Base(name))
  450. return nil
  451. }
  452. func (fs *fakeFS) RemoveAll(name string) error {
  453. fs.mut.Lock()
  454. defer fs.mut.Unlock()
  455. fs.counters.RemoveAll++
  456. time.Sleep(fs.latency)
  457. if fs.insens {
  458. name = UnicodeLowercaseNormalized(name)
  459. }
  460. entry := fs.entryForName(filepath.Dir(name))
  461. if entry == nil {
  462. return nil // all tested real systems exhibit this behaviour
  463. }
  464. // RemoveAll is easy when the file system uses garbage collection under
  465. // the hood... We even get the correct semantics for open fd:s for free.
  466. delete(entry.children, filepath.Base(name))
  467. return nil
  468. }
  469. func (fs *fakeFS) Rename(oldname, newname string) error {
  470. fs.mut.Lock()
  471. defer fs.mut.Unlock()
  472. fs.counters.Rename++
  473. time.Sleep(fs.latency)
  474. oldKey := filepath.Base(oldname)
  475. newKey := filepath.Base(newname)
  476. if fs.insens {
  477. oldKey = UnicodeLowercaseNormalized(oldKey)
  478. newKey = UnicodeLowercaseNormalized(newKey)
  479. }
  480. p0 := fs.entryForName(filepath.Dir(oldname))
  481. if p0 == nil {
  482. return os.ErrNotExist
  483. }
  484. entry := p0.children[oldKey]
  485. if entry == nil {
  486. return os.ErrNotExist
  487. }
  488. p1 := fs.entryForName(filepath.Dir(newname))
  489. if p1 == nil {
  490. return os.ErrNotExist
  491. }
  492. dst, ok := p1.children[newKey]
  493. if ok {
  494. if fs.insens && newKey == oldKey {
  495. // case-only in-place rename
  496. entry.name = filepath.Base(newname)
  497. return nil
  498. }
  499. if dst.entryType == fakeEntryTypeDir {
  500. return errors.New("is a directory")
  501. }
  502. }
  503. p1.children[newKey] = entry
  504. entry.name = filepath.Base(newname)
  505. delete(p0.children, oldKey)
  506. return nil
  507. }
  508. func (fs *fakeFS) Stat(name string) (FileInfo, error) {
  509. return fs.Lstat(name)
  510. }
  511. func (fs *fakeFS) SymlinksSupported() bool {
  512. return false
  513. }
  514. func (fs *fakeFS) Walk(name string, walkFn WalkFunc) error {
  515. return errors.New("not implemented")
  516. }
  517. func (fs *fakeFS) Watch(path string, ignore Matcher, ctx context.Context, ignorePerms bool) (<-chan Event, <-chan error, error) {
  518. return nil, nil, ErrWatchNotSupported
  519. }
  520. func (fs *fakeFS) Hide(name string) error {
  521. return nil
  522. }
  523. func (fs *fakeFS) Unhide(name string) error {
  524. return nil
  525. }
  526. func (fs *fakeFS) Glob(pattern string) ([]string, error) {
  527. // gnnh we don't seem to actually require this in practice
  528. return nil, errors.New("not implemented")
  529. }
  530. func (fs *fakeFS) Roots() ([]string, error) {
  531. return []string{"/"}, nil
  532. }
  533. func (fs *fakeFS) Usage(name string) (Usage, error) {
  534. return Usage{}, errors.New("not implemented")
  535. }
  536. func (fs *fakeFS) Type() FilesystemType {
  537. return FilesystemTypeFake
  538. }
  539. func (fs *fakeFS) URI() string {
  540. return fs.uri
  541. }
  542. func (fs *fakeFS) Options() []Option {
  543. return nil
  544. }
  545. func (fs *fakeFS) SameFile(fi1, fi2 FileInfo) bool {
  546. // BUG: real systems base file sameness on path, inodes, etc
  547. // we try our best, but FileInfo just doesn't have enough data
  548. // so there be false positives, especially on Windows
  549. // where ModTime is not that precise
  550. var ok bool
  551. if fs.insens {
  552. ok = UnicodeLowercaseNormalized(fi1.Name()) == UnicodeLowercaseNormalized(fi2.Name())
  553. } else {
  554. ok = fi1.Name() == fi2.Name()
  555. }
  556. return ok && fi1.ModTime().Equal(fi2.ModTime()) && fi1.Mode() == fi2.Mode() && fi1.IsDir() == fi2.IsDir() && fi1.IsRegular() == fi2.IsRegular() && fi1.IsSymlink() == fi2.IsSymlink() && fi1.Owner() == fi2.Owner() && fi1.Group() == fi2.Group()
  557. }
  558. func (fs *fakeFS) underlying() (Filesystem, bool) {
  559. return nil, false
  560. }
  561. func (fs *fakeFS) wrapperType() filesystemWrapperType {
  562. return filesystemWrapperTypeNone
  563. }
  564. func (fs *fakeFS) resetCounters() {
  565. fs.mut.Lock()
  566. fs.counters = fakeFSCounters{}
  567. fs.mut.Unlock()
  568. }
  569. func (fs *fakeFS) reportMetricsPerOp(b *testing.B) {
  570. fs.reportMetricsPer(b, 1, "op")
  571. }
  572. func (fs *fakeFS) reportMetricsPer(b *testing.B, divisor float64, unit string) {
  573. fs.mut.Lock()
  574. defer fs.mut.Unlock()
  575. b.ReportMetric(float64(fs.counters.Lstat)/divisor/float64(b.N), "Lstat/"+unit)
  576. b.ReportMetric(float64(fs.counters.DirNames)/divisor/float64(b.N), "DirNames/"+unit)
  577. }
  578. // fakeFile is the representation of an open file. We don't care if it's
  579. // opened for reading or writing, it's all good.
  580. type fakeFile struct {
  581. *fakeEntry
  582. mut sync.Mutex
  583. rng io.Reader
  584. seed int64
  585. offset int64
  586. seedOffs int64
  587. presentedName string // present (i.e. != "") on insensitive fs only
  588. }
  589. func (f *fakeFile) Close() error {
  590. return nil
  591. }
  592. func (f *fakeFile) Read(p []byte) (int, error) {
  593. f.mut.Lock()
  594. defer f.mut.Unlock()
  595. return f.readShortAt(p, f.offset)
  596. }
  597. func (f *fakeFile) ReadAt(p []byte, offs int64) (int, error) {
  598. f.mut.Lock()
  599. defer f.mut.Unlock()
  600. // ReadAt is spec:ed to always read a full block unless EOF or failure,
  601. // so we must loop. It's also not supposed to affect the seek position,
  602. // but that would make things annoying or inefficient in terms of
  603. // generating the appropriate RNG etc so I ignore that. In practice we
  604. // currently don't depend on that aspect of it...
  605. var read int
  606. for {
  607. n, err := f.readShortAt(p[read:], offs+int64(read))
  608. read += n
  609. if err != nil {
  610. return read, err
  611. }
  612. if read == len(p) {
  613. return read, nil
  614. }
  615. }
  616. }
  617. func (f *fakeFile) readShortAt(p []byte, offs int64) (int, error) {
  618. // Here be a certain amount of magic... We want to return pseudorandom,
  619. // predictable data so that a read from the same offset in the same file
  620. // always returns the same data. But the RNG is a stream, and reads can
  621. // be random.
  622. //
  623. // We split the file into "blocks" numbered by "seedNo", where each
  624. // block becomes an instantiation of the RNG, seeded with the hash of
  625. // the file number plus the seedNo (block number). We keep the RNG
  626. // around in the hope that the next read will be sequential to this one
  627. // and we can continue reading from the same RNG.
  628. //
  629. // When that's not the case we create a new RNG for the block we are in,
  630. // read as many bytes from it as necessary to get to the right offset,
  631. // and then serve the read from there. We limit the length of the read
  632. // to the end of the block, as another RNG needs to be created to serve
  633. // the next block.
  634. //
  635. // The size of the blocks are a matter of taste... Larger blocks give
  636. // better performance for sequential reads, but worse for random reads
  637. // as we often need to generate and throw away a lot of data at the
  638. // start of the block to serve a given read. 128 KiB blocks fit
  639. // reasonably well with the type of IO Syncthing tends to do.
  640. if f.entryType == fakeEntryTypeDir {
  641. return 0, errors.New("is a directory")
  642. }
  643. if offs >= f.size {
  644. return 0, io.EOF
  645. }
  646. if f.content != nil {
  647. n := copy(p, f.content[int(offs):])
  648. f.offset = offs + int64(n)
  649. return n, nil
  650. }
  651. // Lazily calculate our main seed, a simple 64 bit FNV hash our file
  652. // name.
  653. if f.seed == 0 {
  654. hf := fnv.New64()
  655. hf.Write([]byte(f.name))
  656. f.seed = int64(hf.Sum64())
  657. }
  658. // Check whether the read is a continuation of an RNG we already have or
  659. // we need to set up a new one.
  660. seedNo := offs >> randomBlockShift
  661. minOffs := seedNo << randomBlockShift
  662. nextBlockOffs := (seedNo + 1) << randomBlockShift
  663. if f.rng == nil || f.offset != offs || seedNo != f.seedOffs {
  664. // This is not a straight read continuing from a previous one
  665. f.rng = rand.New(rand.NewSource(f.seed + seedNo))
  666. // If the read is not at the start of the block, discard data
  667. // accordingly.
  668. diff := offs - minOffs
  669. if diff > 0 {
  670. lr := io.LimitReader(f.rng, diff)
  671. io.Copy(io.Discard, lr)
  672. }
  673. f.offset = offs
  674. f.seedOffs = seedNo
  675. }
  676. size := len(p)
  677. // Don't read past the end of the file
  678. if offs+int64(size) > f.size {
  679. size = int(f.size - offs)
  680. }
  681. // Don't read across the block boundary
  682. if offs+int64(size) > nextBlockOffs {
  683. size = int(nextBlockOffs - offs)
  684. }
  685. f.offset += int64(size)
  686. return f.rng.Read(p[:size])
  687. }
  688. func (f *fakeFile) Seek(offset int64, whence int) (int64, error) {
  689. f.mut.Lock()
  690. defer f.mut.Unlock()
  691. if f.entryType == fakeEntryTypeDir {
  692. return 0, errors.New("is a directory")
  693. }
  694. f.rng = nil
  695. switch whence {
  696. case io.SeekCurrent:
  697. f.offset += offset
  698. case io.SeekEnd:
  699. f.offset = f.size - offset
  700. case io.SeekStart:
  701. f.offset = offset
  702. }
  703. if f.offset < 0 {
  704. f.offset = 0
  705. return f.offset, errors.New("seek before start")
  706. }
  707. if f.offset > f.size {
  708. f.offset = f.size
  709. return f.offset, io.EOF
  710. }
  711. return f.offset, nil
  712. }
  713. func (f *fakeFile) Write(p []byte) (int, error) {
  714. f.mut.Lock()
  715. offs := f.offset
  716. f.mut.Unlock()
  717. return f.WriteAt(p, offs)
  718. }
  719. func (f *fakeFile) WriteAt(p []byte, off int64) (int, error) {
  720. f.mut.Lock()
  721. defer f.mut.Unlock()
  722. if f.entryType == fakeEntryTypeDir {
  723. return 0, errors.New("is a directory")
  724. }
  725. if f.content != nil {
  726. if len(f.content) < int(off)+len(p) {
  727. newc := make([]byte, int(off)+len(p))
  728. copy(newc, f.content)
  729. f.content = newc
  730. }
  731. copy(f.content[int(off):], p)
  732. }
  733. f.rng = nil
  734. f.offset = off + int64(len(p))
  735. if f.offset > f.size {
  736. f.size = f.offset
  737. }
  738. return len(p), nil
  739. }
  740. func (f *fakeFile) Name() string {
  741. if f.presentedName != "" {
  742. return f.presentedName
  743. }
  744. return f.name
  745. }
  746. func (f *fakeFile) Truncate(size int64) error {
  747. f.mut.Lock()
  748. defer f.mut.Unlock()
  749. if f.content != nil {
  750. if int64(cap(f.content)) < size {
  751. c := make([]byte, size)
  752. copy(c[:len(f.content)], f.content)
  753. f.content = c
  754. } else {
  755. f.content = f.content[:int(size)]
  756. }
  757. }
  758. f.rng = nil
  759. f.size = size
  760. if f.offset > size {
  761. f.offset = size
  762. }
  763. return nil
  764. }
  765. func (f *fakeFile) Stat() (FileInfo, error) {
  766. info := &fakeFileInfo{*f.fakeEntry}
  767. if f.presentedName != "" {
  768. info.name = f.presentedName
  769. }
  770. return info, nil
  771. }
  772. func (f *fakeFile) Sync() error {
  773. return nil
  774. }
  775. // fakeFileInfo is the stat result.
  776. type fakeFileInfo struct {
  777. fakeEntry // intentionally a copy of the struct
  778. }
  779. func (f *fakeFileInfo) Name() string {
  780. return f.name
  781. }
  782. func (f *fakeFileInfo) Mode() FileMode {
  783. return f.mode
  784. }
  785. func (f *fakeFileInfo) Size() int64 {
  786. return f.size
  787. }
  788. func (f *fakeFileInfo) ModTime() time.Time {
  789. return f.mtime
  790. }
  791. func (f *fakeFileInfo) IsDir() bool {
  792. return f.entryType == fakeEntryTypeDir
  793. }
  794. func (f *fakeFileInfo) IsRegular() bool {
  795. return f.entryType == fakeEntryTypeFile
  796. }
  797. func (f *fakeFileInfo) IsSymlink() bool {
  798. return f.entryType == fakeEntryTypeSymlink
  799. }
  800. func (f *fakeFileInfo) Owner() int {
  801. return f.uid
  802. }
  803. func (f *fakeFileInfo) Group() int {
  804. return f.gid
  805. }