walk_test.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795
  1. // Copyright (C) 2014 The Syncthing Authors.
  2. //
  3. // This Source Code Form is subject to the terms of the Mozilla Public
  4. // License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5. // You can obtain one at https://mozilla.org/MPL/2.0/.
  6. package scanner
  7. import (
  8. "bytes"
  9. "context"
  10. "crypto/rand"
  11. "fmt"
  12. "io"
  13. "io/ioutil"
  14. "os"
  15. "path/filepath"
  16. "runtime"
  17. rdebug "runtime/debug"
  18. "sort"
  19. "sync"
  20. "testing"
  21. "github.com/d4l3k/messagediff"
  22. "github.com/syncthing/syncthing/lib/fs"
  23. "github.com/syncthing/syncthing/lib/ignore"
  24. "github.com/syncthing/syncthing/lib/osutil"
  25. "github.com/syncthing/syncthing/lib/protocol"
  26. "github.com/syncthing/syncthing/lib/sha256"
  27. "golang.org/x/text/unicode/norm"
  28. )
  29. type testfile struct {
  30. name string
  31. length int64
  32. hash string
  33. }
  34. type testfileList []testfile
  35. var testFs fs.Filesystem
  36. var testdata = testfileList{
  37. {"afile", 4, "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c"},
  38. {"dir1", 128, ""},
  39. {filepath.Join("dir1", "dfile"), 5, "49ae93732fcf8d63fe1cce759664982dbd5b23161f007dba8561862adc96d063"},
  40. {"dir2", 128, ""},
  41. {filepath.Join("dir2", "cfile"), 4, "bf07a7fbb825fc0aae7bf4a1177b2b31fcf8a3feeaf7092761e18c859ee52a9c"},
  42. {"excludes", 37, "df90b52f0c55dba7a7a940affe482571563b1ac57bd5be4d8a0291e7de928e06"},
  43. {"further-excludes", 5, "7eb0a548094fa6295f7fd9200d69973e5f5ec5c04f2a86d998080ac43ecf89f1"},
  44. }
  45. func init() {
  46. // This test runs the risk of entering infinite recursion if it fails.
  47. // Limit the stack size to 10 megs to crash early in that case instead of
  48. // potentially taking down the box...
  49. rdebug.SetMaxStack(10 * 1 << 20)
  50. testFs = fs.NewFilesystem(fs.FilesystemTypeBasic, "testdata")
  51. }
  52. func TestWalkSub(t *testing.T) {
  53. ignores := ignore.New(testFs)
  54. err := ignores.Load(".stignore")
  55. if err != nil {
  56. t.Fatal(err)
  57. }
  58. fchan := Walk(context.TODO(), Config{
  59. Filesystem: testFs,
  60. Subs: []string{"dir2"},
  61. Matcher: ignores,
  62. Hashers: 2,
  63. })
  64. var files []protocol.FileInfo
  65. for f := range fchan {
  66. if f.Err != nil {
  67. t.Errorf("Error while scanning %v: %v", f.Err, f.Path)
  68. }
  69. files = append(files, f.File)
  70. }
  71. // The directory contains two files, where one is ignored from a higher
  72. // level. We should see only the directory and one of the files.
  73. if len(files) != 2 {
  74. t.Fatalf("Incorrect length %d != 2", len(files))
  75. }
  76. if files[0].Name != "dir2" {
  77. t.Errorf("Incorrect file %v != dir2", files[0])
  78. }
  79. if files[1].Name != filepath.Join("dir2", "cfile") {
  80. t.Errorf("Incorrect file %v != dir2/cfile", files[1])
  81. }
  82. }
  83. func TestWalk(t *testing.T) {
  84. ignores := ignore.New(testFs)
  85. err := ignores.Load(".stignore")
  86. if err != nil {
  87. t.Fatal(err)
  88. }
  89. t.Log(ignores)
  90. fchan := Walk(context.TODO(), Config{
  91. Filesystem: testFs,
  92. Matcher: ignores,
  93. Hashers: 2,
  94. })
  95. var tmp []protocol.FileInfo
  96. for f := range fchan {
  97. if f.Err != nil {
  98. t.Errorf("Error while scanning %v: %v", f.Err, f.Path)
  99. }
  100. tmp = append(tmp, f.File)
  101. }
  102. sort.Sort(fileList(tmp))
  103. files := fileList(tmp).testfiles()
  104. if diff, equal := messagediff.PrettyDiff(testdata, files); !equal {
  105. t.Errorf("Walk returned unexpected data. Diff:\n%s", diff)
  106. }
  107. }
  108. func TestVerify(t *testing.T) {
  109. blocksize := 16
  110. // data should be an even multiple of blocksize long
  111. data := []byte("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut e")
  112. buf := bytes.NewBuffer(data)
  113. progress := newByteCounter()
  114. defer progress.Close()
  115. blocks, err := Blocks(context.TODO(), buf, blocksize, -1, progress, false)
  116. if err != nil {
  117. t.Fatal(err)
  118. }
  119. if exp := len(data) / blocksize; len(blocks) != exp {
  120. t.Fatalf("Incorrect number of blocks %d != %d", len(blocks), exp)
  121. }
  122. if int64(len(data)) != progress.Total() {
  123. t.Fatalf("Incorrect counter value %d != %d", len(data), progress.Total())
  124. }
  125. buf = bytes.NewBuffer(data)
  126. err = verify(buf, blocksize, blocks)
  127. t.Log(err)
  128. if err != nil {
  129. t.Fatal("Unexpected verify failure", err)
  130. }
  131. buf = bytes.NewBuffer(append(data, '\n'))
  132. err = verify(buf, blocksize, blocks)
  133. t.Log(err)
  134. if err == nil {
  135. t.Fatal("Unexpected verify success")
  136. }
  137. buf = bytes.NewBuffer(data[:len(data)-1])
  138. err = verify(buf, blocksize, blocks)
  139. t.Log(err)
  140. if err == nil {
  141. t.Fatal("Unexpected verify success")
  142. }
  143. data[42] = 42
  144. buf = bytes.NewBuffer(data)
  145. err = verify(buf, blocksize, blocks)
  146. t.Log(err)
  147. if err == nil {
  148. t.Fatal("Unexpected verify success")
  149. }
  150. }
  151. func TestNormalization(t *testing.T) {
  152. if runtime.GOOS == "darwin" {
  153. t.Skip("Normalization test not possible on darwin")
  154. return
  155. }
  156. os.RemoveAll("testdata/normalization")
  157. defer os.RemoveAll("testdata/normalization")
  158. tests := []string{
  159. "0-A", // ASCII A -- accepted
  160. "1-\xC3\x84", // NFC 'Ä' -- conflicts with the entry below, accepted
  161. "1-\x41\xCC\x88", // NFD 'Ä' -- conflicts with the entry above, ignored
  162. "2-\xC3\x85", // NFC 'Å' -- accepted
  163. "3-\x41\xCC\x83", // NFD 'Ã' -- converted to NFC
  164. "4-\xE2\x98\x95", // U+2615 HOT BEVERAGE (☕) -- accepted
  165. "5-\xCD\xE2", // EUC-CN "wài" (外) -- ignored (not UTF8)
  166. }
  167. numInvalid := 2
  168. if runtime.GOOS == "windows" {
  169. // On Windows, in case 5 the character gets replaced with a
  170. // replacement character \xEF\xBF\xBD at the point it's written to disk,
  171. // which means it suddenly becomes valid (sort of).
  172. numInvalid--
  173. }
  174. numValid := len(tests) - numInvalid
  175. for _, s1 := range tests {
  176. // Create a directory for each of the interesting strings above
  177. if err := testFs.MkdirAll(filepath.Join("normalization", s1), 0755); err != nil {
  178. t.Fatal(err)
  179. }
  180. for _, s2 := range tests {
  181. // Within each dir, create a file with each of the interesting
  182. // file names. Ensure that the file doesn't exist when it's
  183. // created. This detects and fails if there's file name
  184. // normalization stuff at the filesystem level.
  185. if fd, err := testFs.OpenFile(filepath.Join("normalization", s1, s2), os.O_CREATE|os.O_EXCL, 0644); err != nil {
  186. t.Fatal(err)
  187. } else {
  188. fd.Write([]byte("test"))
  189. fd.Close()
  190. }
  191. }
  192. }
  193. // We can normalize a directory name, but we can't descend into it in the
  194. // same pass due to how filepath.Walk works. So we run the scan twice to
  195. // make sure it all gets done. In production, things will be correct
  196. // eventually...
  197. walkDir(testFs, "normalization", nil, nil, 0)
  198. tmp := walkDir(testFs, "normalization", nil, nil, 0)
  199. files := fileList(tmp).testfiles()
  200. // We should have one file per combination, plus the directories
  201. // themselves, plus the "testdata/normalization" directory
  202. expectedNum := numValid*numValid + numValid + 1
  203. if len(files) != expectedNum {
  204. t.Errorf("Expected %d files, got %d", expectedNum, len(files))
  205. }
  206. // The file names should all be in NFC form.
  207. for _, f := range files {
  208. t.Logf("%q (% x) %v", f.name, f.name, norm.NFC.IsNormalString(f.name))
  209. if !norm.NFC.IsNormalString(f.name) {
  210. t.Errorf("File name %q is not NFC normalized", f.name)
  211. }
  212. }
  213. }
  214. func TestIssue1507(t *testing.T) {
  215. w := &walker{}
  216. w.Matcher = ignore.New(w.Filesystem)
  217. h := make(chan protocol.FileInfo, 100)
  218. f := make(chan ScanResult, 100)
  219. fn := w.walkAndHashFiles(context.TODO(), h, f)
  220. fn("", nil, protocol.ErrClosed)
  221. }
  222. func TestWalkSymlinkUnix(t *testing.T) {
  223. if runtime.GOOS == "windows" {
  224. t.Skip("skipping unsupported symlink test")
  225. return
  226. }
  227. // Create a folder with a symlink in it
  228. os.RemoveAll("_symlinks")
  229. os.Mkdir("_symlinks", 0755)
  230. defer os.RemoveAll("_symlinks")
  231. os.Symlink("../testdata", "_symlinks/link")
  232. fs := fs.NewFilesystem(fs.FilesystemTypeBasic, "_symlinks")
  233. for _, path := range []string{".", "link"} {
  234. // Scan it
  235. files := walkDir(fs, path, nil, nil, 0)
  236. // Verify that we got one symlink and with the correct attributes
  237. if len(files) != 1 {
  238. t.Errorf("expected 1 symlink, not %d", len(files))
  239. }
  240. if len(files[0].Blocks) != 0 {
  241. t.Errorf("expected zero blocks for symlink, not %d", len(files[0].Blocks))
  242. }
  243. if files[0].SymlinkTarget != "../testdata" {
  244. t.Errorf("expected symlink to have target destination, not %q", files[0].SymlinkTarget)
  245. }
  246. }
  247. }
  248. func TestWalkSymlinkWindows(t *testing.T) {
  249. if runtime.GOOS != "windows" {
  250. t.Skip("skipping unsupported symlink test")
  251. }
  252. // Create a folder with a symlink in it
  253. name := "_symlinks-win"
  254. os.RemoveAll(name)
  255. os.Mkdir(name, 0755)
  256. defer os.RemoveAll(name)
  257. fs := fs.NewFilesystem(fs.FilesystemTypeBasic, name)
  258. if err := osutil.DebugSymlinkForTestsOnly("../testdata", "_symlinks/link"); err != nil {
  259. // Probably we require permissions we don't have.
  260. t.Skip(err)
  261. }
  262. for _, path := range []string{".", "link"} {
  263. // Scan it
  264. files := walkDir(fs, path, nil, nil, 0)
  265. // Verify that we got zero symlinks
  266. if len(files) != 0 {
  267. t.Errorf("expected zero symlinks, not %d", len(files))
  268. }
  269. }
  270. }
  271. func TestWalkRootSymlink(t *testing.T) {
  272. // Create a folder with a symlink in it
  273. tmp, err := ioutil.TempDir("", "")
  274. if err != nil {
  275. t.Fatal(err)
  276. }
  277. defer os.RemoveAll(tmp)
  278. link := tmp + "/link"
  279. dest, _ := filepath.Abs("testdata/dir1")
  280. if err := osutil.DebugSymlinkForTestsOnly(dest, link); err != nil {
  281. if runtime.GOOS == "windows" {
  282. // Probably we require permissions we don't have.
  283. t.Skip("Need admin permissions or developer mode to run symlink test on Windows: " + err.Error())
  284. } else {
  285. t.Fatal(err)
  286. }
  287. }
  288. // Scan it
  289. files := walkDir(fs.NewFilesystem(fs.FilesystemTypeBasic, link), ".", nil, nil, 0)
  290. // Verify that we got two files
  291. if len(files) != 2 {
  292. t.Errorf("expected two files, not %d", len(files))
  293. }
  294. }
  295. func TestBlocksizeHysteresis(t *testing.T) {
  296. // Verify that we select the right block size in the presence of old
  297. // file information.
  298. if testing.Short() {
  299. t.Skip("long and hard test")
  300. }
  301. sf := fs.NewWalkFilesystem(&singleFileFS{
  302. name: "testfile.dat",
  303. filesize: 500 << 20, // 500 MiB
  304. })
  305. current := make(fakeCurrentFiler)
  306. runTest := func(expectedBlockSize int) {
  307. files := walkDir(sf, ".", current, nil, 0)
  308. if len(files) != 1 {
  309. t.Fatalf("expected one file, not %d", len(files))
  310. }
  311. if s := files[0].BlockSize(); s != expectedBlockSize {
  312. t.Fatalf("incorrect block size %d != expected %d", s, expectedBlockSize)
  313. }
  314. }
  315. // Scan with no previous knowledge. We should get a 512 KiB block size.
  316. runTest(512 << 10)
  317. // Scan on the assumption that previous size was 256 KiB. Retain 256 KiB
  318. // block size.
  319. current["testfile.dat"] = protocol.FileInfo{
  320. Name: "testfile.dat",
  321. Size: 500 << 20,
  322. RawBlockSize: 256 << 10,
  323. }
  324. runTest(256 << 10)
  325. // Scan on the assumption that previous size was 1 MiB. Retain 1 MiB
  326. // block size.
  327. current["testfile.dat"] = protocol.FileInfo{
  328. Name: "testfile.dat",
  329. Size: 500 << 20,
  330. RawBlockSize: 1 << 20,
  331. }
  332. runTest(1 << 20)
  333. // Scan on the assumption that previous size was 128 KiB. Move to 512
  334. // KiB because the difference is large.
  335. current["testfile.dat"] = protocol.FileInfo{
  336. Name: "testfile.dat",
  337. Size: 500 << 20,
  338. RawBlockSize: 128 << 10,
  339. }
  340. runTest(512 << 10)
  341. // Scan on the assumption that previous size was 2 MiB. Move to 512
  342. // KiB because the difference is large.
  343. current["testfile.dat"] = protocol.FileInfo{
  344. Name: "testfile.dat",
  345. Size: 500 << 20,
  346. RawBlockSize: 2 << 20,
  347. }
  348. runTest(512 << 10)
  349. }
  350. func TestWalkReceiveOnly(t *testing.T) {
  351. sf := fs.NewWalkFilesystem(&singleFileFS{
  352. name: "testfile.dat",
  353. filesize: 1024,
  354. })
  355. current := make(fakeCurrentFiler)
  356. // Initial scan, no files in the CurrentFiler. Should pick up the file and
  357. // set the ReceiveOnly flag on it, because that's the flag we give the
  358. // walker to set.
  359. files := walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
  360. if len(files) != 1 {
  361. t.Fatal("Should have scanned one file")
  362. }
  363. if files[0].LocalFlags != protocol.FlagLocalReceiveOnly {
  364. t.Fatal("Should have set the ReceiveOnly flag")
  365. }
  366. // Update the CurrentFiler and scan again. It should not return
  367. // anything, because the file has not changed. This verifies that the
  368. // ReceiveOnly flag is properly ignored and doesn't trigger a rescan
  369. // every time.
  370. cur := files[0]
  371. current[cur.Name] = cur
  372. files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
  373. if len(files) != 0 {
  374. t.Fatal("Should not have scanned anything")
  375. }
  376. // Now pretend the file was previously ignored instead. We should pick up
  377. // the difference in flags and set just the LocalReceive flags.
  378. cur.LocalFlags = protocol.FlagLocalIgnored
  379. current[cur.Name] = cur
  380. files = walkDir(sf, ".", current, nil, protocol.FlagLocalReceiveOnly)
  381. if len(files) != 1 {
  382. t.Fatal("Should have scanned one file")
  383. }
  384. if files[0].LocalFlags != protocol.FlagLocalReceiveOnly {
  385. t.Fatal("Should have set the ReceiveOnly flag")
  386. }
  387. }
  388. func walkDir(fs fs.Filesystem, dir string, cfiler CurrentFiler, matcher *ignore.Matcher, localFlags uint32) []protocol.FileInfo {
  389. fchan := Walk(context.TODO(), Config{
  390. Filesystem: fs,
  391. Subs: []string{dir},
  392. AutoNormalize: true,
  393. Hashers: 2,
  394. CurrentFiler: cfiler,
  395. Matcher: matcher,
  396. LocalFlags: localFlags,
  397. })
  398. var tmp []protocol.FileInfo
  399. for f := range fchan {
  400. if f.Err == nil {
  401. tmp = append(tmp, f.File)
  402. }
  403. }
  404. sort.Sort(fileList(tmp))
  405. return tmp
  406. }
  407. type fileList []protocol.FileInfo
  408. func (l fileList) Len() int {
  409. return len(l)
  410. }
  411. func (l fileList) Less(a, b int) bool {
  412. return l[a].Name < l[b].Name
  413. }
  414. func (l fileList) Swap(a, b int) {
  415. l[a], l[b] = l[b], l[a]
  416. }
  417. func (l fileList) testfiles() testfileList {
  418. testfiles := make(testfileList, len(l))
  419. for i, f := range l {
  420. if len(f.Blocks) > 1 {
  421. panic("simple test case stuff only supports a single block per file")
  422. }
  423. testfiles[i] = testfile{name: f.Name, length: f.FileSize()}
  424. if len(f.Blocks) == 1 {
  425. testfiles[i].hash = fmt.Sprintf("%x", f.Blocks[0].Hash)
  426. }
  427. }
  428. return testfiles
  429. }
  430. func (l testfileList) String() string {
  431. var b bytes.Buffer
  432. b.WriteString("{\n")
  433. for _, f := range l {
  434. fmt.Fprintf(&b, " %s (%d bytes): %s\n", f.name, f.length, f.hash)
  435. }
  436. b.WriteString("}")
  437. return b.String()
  438. }
  439. var initOnce sync.Once
  440. const (
  441. testdataSize = 17 << 20
  442. testdataName = "_random.data"
  443. )
  444. func BenchmarkHashFile(b *testing.B) {
  445. initOnce.Do(initTestFile)
  446. b.ResetTimer()
  447. for i := 0; i < b.N; i++ {
  448. if _, err := HashFile(context.TODO(), fs.NewFilesystem(fs.FilesystemTypeBasic, ""), testdataName, protocol.MinBlockSize, nil, true); err != nil {
  449. b.Fatal(err)
  450. }
  451. }
  452. b.SetBytes(testdataSize)
  453. b.ReportAllocs()
  454. }
  455. func initTestFile() {
  456. fd, err := os.Create(testdataName)
  457. if err != nil {
  458. panic(err)
  459. }
  460. lr := io.LimitReader(rand.Reader, testdataSize)
  461. if _, err := io.Copy(fd, lr); err != nil {
  462. panic(err)
  463. }
  464. if err := fd.Close(); err != nil {
  465. panic(err)
  466. }
  467. }
  468. func TestStopWalk(t *testing.T) {
  469. // Create tree that is 100 levels deep, with each level containing 100
  470. // files (each 1 MB) and 100 directories (in turn containing 100 files
  471. // and 100 directories, etc). That is, in total > 100^100 files and as
  472. // many directories. It'll take a while to scan, giving us time to
  473. // cancel it and make sure the scan stops.
  474. // Use an errorFs as the backing fs for the rest of the interface
  475. // The way we get it is a bit hacky tho.
  476. errorFs := fs.NewFilesystem(fs.FilesystemType(-1), ".")
  477. fs := fs.NewWalkFilesystem(&infiniteFS{errorFs, 100, 100, 1e6})
  478. const numHashers = 4
  479. ctx, cancel := context.WithCancel(context.Background())
  480. fchan := Walk(ctx, Config{
  481. Filesystem: fs,
  482. Hashers: numHashers,
  483. ProgressTickIntervalS: -1, // Don't attempt to build the full list of files before starting to scan...
  484. })
  485. // Receive a few entries to make sure the walker is up and running,
  486. // scanning both files and dirs. Do some quick sanity tests on the
  487. // returned file entries to make sure we are not just reading crap from
  488. // a closed channel or something.
  489. dirs := 0
  490. files := 0
  491. for {
  492. res := <-fchan
  493. if res.Err != nil {
  494. t.Errorf("Error while scanning %v: %v", res.Err, res.Path)
  495. }
  496. f := res.File
  497. t.Log("Scanned", f)
  498. if f.IsDirectory() {
  499. if len(f.Name) == 0 || f.Permissions == 0 {
  500. t.Error("Bad directory entry", f)
  501. }
  502. dirs++
  503. } else {
  504. if len(f.Name) == 0 || len(f.Blocks) == 0 || f.Permissions == 0 {
  505. t.Error("Bad file entry", f)
  506. }
  507. files++
  508. }
  509. if dirs > 5 && files > 5 {
  510. break
  511. }
  512. }
  513. // Cancel the walker.
  514. cancel()
  515. // Empty out any waiting entries and wait for the channel to close.
  516. // Count them, they should be zero or very few - essentially, each
  517. // hasher has the choice of returning a fully handled entry or
  518. // cancelling, but they should not start on another item.
  519. extra := 0
  520. for range fchan {
  521. extra++
  522. }
  523. t.Log("Extra entries:", extra)
  524. if extra > numHashers {
  525. t.Error("unexpected extra entries received after cancel")
  526. }
  527. }
  528. func TestIssue4799(t *testing.T) {
  529. tmp, err := ioutil.TempDir("", "")
  530. if err != nil {
  531. t.Fatal(err)
  532. }
  533. defer os.RemoveAll(tmp)
  534. fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
  535. fd, err := fs.Create("foo")
  536. if err != nil {
  537. t.Fatal(err)
  538. }
  539. fd.Close()
  540. files := walkDir(fs, "/foo", nil, nil, 0)
  541. if len(files) != 1 || files[0].Name != "foo" {
  542. t.Error(`Received unexpected file infos when walking "/foo"`, files)
  543. }
  544. }
  545. func TestRecurseInclude(t *testing.T) {
  546. stignore := `
  547. !/dir1/cfile
  548. !efile
  549. !ffile
  550. *
  551. `
  552. ignores := ignore.New(testFs, ignore.WithCache(true))
  553. if err := ignores.Parse(bytes.NewBufferString(stignore), ".stignore"); err != nil {
  554. t.Fatal(err)
  555. }
  556. files := walkDir(testFs, ".", nil, ignores, 0)
  557. expected := []string{
  558. filepath.Join("dir1"),
  559. filepath.Join("dir1", "cfile"),
  560. filepath.Join("dir2"),
  561. filepath.Join("dir2", "dir21"),
  562. filepath.Join("dir2", "dir21", "dir22"),
  563. filepath.Join("dir2", "dir21", "dir22", "dir23"),
  564. filepath.Join("dir2", "dir21", "dir22", "dir23", "efile"),
  565. filepath.Join("dir2", "dir21", "dir22", "efile"),
  566. filepath.Join("dir2", "dir21", "dir22", "efile", "efile"),
  567. filepath.Join("dir2", "dir21", "dira"),
  568. filepath.Join("dir2", "dir21", "dira", "efile"),
  569. filepath.Join("dir2", "dir21", "dira", "ffile"),
  570. filepath.Join("dir2", "dir21", "efile"),
  571. filepath.Join("dir2", "dir21", "efile", "ign"),
  572. filepath.Join("dir2", "dir21", "efile", "ign", "efile"),
  573. }
  574. if len(files) != len(expected) {
  575. t.Fatalf("Got %d files %v, expected %d files at %v", len(files), files, len(expected), expected)
  576. }
  577. for i := range files {
  578. if files[i].Name != expected[i] {
  579. t.Errorf("Got %v, expected file at %v", files[i], expected[i])
  580. }
  581. }
  582. }
  583. func TestIssue4841(t *testing.T) {
  584. tmp, err := ioutil.TempDir("", "")
  585. if err != nil {
  586. t.Fatal(err)
  587. }
  588. defer os.RemoveAll(tmp)
  589. fs := fs.NewFilesystem(fs.FilesystemTypeBasic, tmp)
  590. fd, err := fs.Create("foo")
  591. if err != nil {
  592. panic(err)
  593. }
  594. fd.Close()
  595. fchan := Walk(context.TODO(), Config{
  596. Filesystem: fs,
  597. Subs: nil,
  598. AutoNormalize: true,
  599. Hashers: 2,
  600. CurrentFiler: fakeCurrentFiler{
  601. "foo": {
  602. Name: "foo",
  603. Type: protocol.FileInfoTypeFile,
  604. LocalFlags: protocol.FlagLocalIgnored,
  605. Version: protocol.Vector{}.Update(1),
  606. },
  607. },
  608. ShortID: protocol.LocalDeviceID.Short(),
  609. })
  610. var files []protocol.FileInfo
  611. for f := range fchan {
  612. if f.Err != nil {
  613. t.Errorf("Error while scanning %v: %v", f.Err, f.Path)
  614. }
  615. files = append(files, f.File)
  616. }
  617. sort.Sort(fileList(files))
  618. if len(files) != 1 {
  619. t.Fatalf("Expected 1 file, got %d: %v", len(files), files)
  620. }
  621. if expected := (protocol.Vector{}.Update(protocol.LocalDeviceID.Short())); !files[0].Version.Equal(expected) {
  622. t.Fatalf("Expected Version == %v, got %v", expected, files[0].Version)
  623. }
  624. }
  625. // TestNotExistingError reproduces https://github.com/syncthing/syncthing/issues/5385
  626. func TestNotExistingError(t *testing.T) {
  627. sub := "notExisting"
  628. if _, err := testFs.Lstat(sub); !fs.IsNotExist(err) {
  629. t.Fatalf("Lstat returned error %v, while nothing should exist there.", err)
  630. }
  631. fchan := Walk(context.TODO(), Config{
  632. Filesystem: testFs,
  633. Subs: []string{sub},
  634. Hashers: 2,
  635. })
  636. for f := range fchan {
  637. t.Fatalf("Expected no result from scan, got %v", f)
  638. }
  639. }
  640. // Verify returns nil or an error describing the mismatch between the block
  641. // list and actual reader contents
  642. func verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
  643. hf := sha256.New()
  644. // A 32k buffer is used for copying into the hash function.
  645. buf := make([]byte, 32<<10)
  646. for i, block := range blocks {
  647. lr := &io.LimitedReader{R: r, N: int64(blocksize)}
  648. _, err := io.CopyBuffer(hf, lr, buf)
  649. if err != nil {
  650. return err
  651. }
  652. hash := hf.Sum(nil)
  653. hf.Reset()
  654. if !bytes.Equal(hash, block.Hash) {
  655. return fmt.Errorf("hash mismatch %x != %x for block %d", hash, block.Hash, i)
  656. }
  657. }
  658. // We should have reached the end now
  659. bs := make([]byte, 1)
  660. n, err := r.Read(bs)
  661. if n != 0 || err != io.EOF {
  662. return fmt.Errorf("file continues past end of blocks")
  663. }
  664. return nil
  665. }
  666. type fakeCurrentFiler map[string]protocol.FileInfo
  667. func (fcf fakeCurrentFiler) CurrentFile(name string) (protocol.FileInfo, bool) {
  668. f, ok := fcf[name]
  669. return f, ok
  670. }