walk.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761
  1. // Copyright (C) 2014 The Syncthing Authors.
  2. //
  3. // This Source Code Form is subject to the terms of the Mozilla Public
  4. // License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5. // You can obtain one at https://mozilla.org/MPL/2.0/.
  6. package scanner
  7. import (
  8. "context"
  9. "errors"
  10. "fmt"
  11. "path/filepath"
  12. "strings"
  13. "sync/atomic"
  14. "time"
  15. "unicode/utf8"
  16. metrics "github.com/rcrowley/go-metrics"
  17. "golang.org/x/text/unicode/norm"
  18. "github.com/syncthing/syncthing/lib/build"
  19. "github.com/syncthing/syncthing/lib/events"
  20. "github.com/syncthing/syncthing/lib/fs"
  21. "github.com/syncthing/syncthing/lib/ignore"
  22. "github.com/syncthing/syncthing/lib/osutil"
  23. "github.com/syncthing/syncthing/lib/protocol"
  24. )
  25. type Config struct {
  26. // Folder for which the walker has been created
  27. Folder string
  28. // Limit walking to these paths within Dir, or no limit if Sub is empty
  29. Subs []string
  30. // If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
  31. Matcher *ignore.Matcher
  32. // Number of hours to keep temporary files for
  33. TempLifetime time.Duration
  34. // If CurrentFiler is not nil, it is queried for the current file before rescanning.
  35. CurrentFiler CurrentFiler
  36. // The Filesystem provides an abstraction on top of the actual filesystem.
  37. Filesystem fs.Filesystem
  38. // If IgnorePerms is true, changes to permission bits will not be
  39. // detected.
  40. IgnorePerms bool
  41. // When AutoNormalize is set, file names that are in UTF8 but incorrect
  42. // normalization form will be corrected.
  43. AutoNormalize bool
  44. // Number of routines to use for hashing
  45. Hashers int
  46. // Our vector clock id
  47. ShortID protocol.ShortID
  48. // Optional progress tick interval which defines how often FolderScanProgress
  49. // events are emitted. Negative number means disabled.
  50. ProgressTickIntervalS int
  51. // Local flags to set on scanned files
  52. LocalFlags uint32
  53. // Modification time is to be considered unchanged if the difference is lower.
  54. ModTimeWindow time.Duration
  55. // Event logger to which the scan progress events are sent
  56. EventLogger events.Logger
  57. // If ScanOwnership is true, we pick up ownership information on files while scanning.
  58. ScanOwnership bool
  59. // If ScanXattrs is true, we pick up extended attributes on files while scanning.
  60. ScanXattrs bool
  61. // Filter for extended attributes
  62. XattrFilter XattrFilter
  63. }
  64. type CurrentFiler interface {
  65. // CurrentFile returns the file as seen at last scan.
  66. CurrentFile(name string) (protocol.FileInfo, bool)
  67. }
  68. type XattrFilter interface {
  69. Permit(string) bool
  70. GetMaxSingleEntrySize() int
  71. GetMaxTotalSize() int
  72. }
  73. type ScanResult struct {
  74. File protocol.FileInfo
  75. Err error
  76. Path string // to be set in case Err != nil and File == nil
  77. }
  78. func Walk(ctx context.Context, cfg Config) chan ScanResult {
  79. return newWalker(cfg).walk(ctx)
  80. }
  81. func WalkWithoutHashing(ctx context.Context, cfg Config) chan ScanResult {
  82. return newWalker(cfg).walkWithoutHashing(ctx)
  83. }
  84. func newWalker(cfg Config) *walker {
  85. w := &walker{cfg}
  86. if w.CurrentFiler == nil {
  87. w.CurrentFiler = noCurrentFiler{}
  88. }
  89. if w.Filesystem == nil {
  90. panic("no filesystem specified")
  91. }
  92. if w.Matcher == nil {
  93. w.Matcher = ignore.New(w.Filesystem)
  94. }
  95. registerFolderMetrics(w.Folder)
  96. return w
  97. }
  98. var (
  99. errUTF8Invalid = errors.New("item is not in UTF8 encoding")
  100. errUTF8Normalization = errors.New("item is not in the correct UTF8 normalization form")
  101. errUTF8Conflict = errors.New("item has UTF8 encoding conflict with another item")
  102. )
  103. type walker struct {
  104. Config
  105. }
  106. // Walk returns the list of files found in the local folder by scanning the
  107. // file system. Files are blockwise hashed.
  108. func (w *walker) walk(ctx context.Context) chan ScanResult {
  109. l.Debugln(w, "Walk", w.Subs, w.Matcher)
  110. toHashChan := make(chan protocol.FileInfo)
  111. finishedChan := make(chan ScanResult)
  112. // A routine which walks the filesystem tree, and sends files which have
  113. // been modified to the counter routine.
  114. go w.scan(ctx, toHashChan, finishedChan)
  115. // We're not required to emit scan progress events, just kick off hashers,
  116. // and feed inputs directly from the walker.
  117. if w.ProgressTickIntervalS < 0 {
  118. newParallelHasher(ctx, w.Folder, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil)
  119. return finishedChan
  120. }
  121. // Defaults to every 2 seconds.
  122. if w.ProgressTickIntervalS == 0 {
  123. w.ProgressTickIntervalS = 2
  124. }
  125. // We need to emit progress events, hence we create a routine which buffers
  126. // the list of files to be hashed, counts the total number of
  127. // bytes to hash, and once no more files need to be hashed (chan gets closed),
  128. // start a routine which periodically emits FolderScanProgress events,
  129. // until a stop signal is sent by the parallel hasher.
  130. // Parallel hasher is stopped by this routine when we close the channel over
  131. // which it receives the files we ask it to hash.
  132. go func() {
  133. var filesToHash []protocol.FileInfo
  134. var total int64 = 1
  135. for file := range toHashChan {
  136. filesToHash = append(filesToHash, file)
  137. total += file.Size
  138. }
  139. if len(filesToHash) == 0 {
  140. close(finishedChan)
  141. return
  142. }
  143. realToHashChan := make(chan protocol.FileInfo)
  144. done := make(chan struct{})
  145. progress := newByteCounter()
  146. newParallelHasher(ctx, w.Folder, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done)
  147. // A routine which actually emits the FolderScanProgress events
  148. // every w.ProgressTicker ticks, until the hasher routines terminate.
  149. go func() {
  150. defer progress.Close()
  151. emitProgressEvent := func() {
  152. current := progress.Total()
  153. rate := progress.Rate()
  154. l.Debugf("%v: Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w, w.Folder, w.Subs, current, total, rate/1024/1024, current*100/total)
  155. w.EventLogger.Log(events.FolderScanProgress, map[string]interface{}{
  156. "folder": w.Folder,
  157. "current": current,
  158. "total": total,
  159. "rate": rate, // bytes per second
  160. })
  161. }
  162. ticker := time.NewTicker(time.Duration(w.ProgressTickIntervalS) * time.Second)
  163. defer ticker.Stop()
  164. for {
  165. select {
  166. case <-done:
  167. emitProgressEvent()
  168. l.Debugln(w, "Walk progress done", w.Folder, w.Subs, w.Matcher)
  169. return
  170. case <-ticker.C:
  171. emitProgressEvent()
  172. case <-ctx.Done():
  173. return
  174. }
  175. }
  176. }()
  177. loop:
  178. for _, file := range filesToHash {
  179. l.Debugln(w, "real to hash:", file.Name)
  180. select {
  181. case realToHashChan <- file:
  182. case <-ctx.Done():
  183. break loop
  184. }
  185. }
  186. close(realToHashChan)
  187. }()
  188. return finishedChan
  189. }
  190. func (w *walker) walkWithoutHashing(ctx context.Context) chan ScanResult {
  191. l.Debugln(w, "Walk without hashing", w.Subs, w.Matcher)
  192. toHashChan := make(chan protocol.FileInfo)
  193. finishedChan := make(chan ScanResult)
  194. // A routine which walks the filesystem tree, and sends files which have
  195. // been modified to the counter routine.
  196. go w.scan(ctx, toHashChan, finishedChan)
  197. go func() {
  198. for file := range toHashChan {
  199. finishedChan <- ScanResult{File: file}
  200. }
  201. close(finishedChan)
  202. }()
  203. return finishedChan
  204. }
  205. const walkFailureEventDesc = "Unexpected error while walking the filesystem during scan"
  206. func (w *walker) scan(ctx context.Context, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) {
  207. hashFiles := w.walkAndHashFiles(ctx, toHashChan, finishedChan)
  208. if len(w.Subs) == 0 {
  209. if err := w.Filesystem.Walk(".", hashFiles); err != nil && !errors.Is(err, fs.SkipDir) {
  210. w.EventLogger.Log(events.Failure, walkFailureEventDesc)
  211. l.Warnf("Aborted scan due to an unexpected error: %v", err)
  212. }
  213. } else {
  214. for _, sub := range w.Subs {
  215. if err := osutil.TraversesSymlink(w.Filesystem, filepath.Dir(sub)); err != nil {
  216. l.Debugf("%v: Skip walking %v as it is below a symlink", w, sub)
  217. continue
  218. }
  219. if err := w.Filesystem.Walk(sub, hashFiles); err != nil && !errors.Is(err, fs.SkipDir) {
  220. w.EventLogger.Log(events.Failure, walkFailureEventDesc)
  221. l.Warnf("Aborted scan of path '%v' due to an unexpected error: %v", sub, err)
  222. }
  223. }
  224. }
  225. close(toHashChan)
  226. }
  227. func (w *walker) walkAndHashFiles(ctx context.Context, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) fs.WalkFunc {
  228. now := time.Now()
  229. ignoredParent := ""
  230. return func(path string, info fs.FileInfo, err error) error {
  231. select {
  232. case <-ctx.Done():
  233. return ctx.Err()
  234. default:
  235. }
  236. metricScannedItems.WithLabelValues(w.Folder).Inc()
  237. // Return value used when we are returning early and don't want to
  238. // process the item. For directories, this means do-not-descend.
  239. var skip error // nil
  240. // info nil when error is not nil
  241. if info != nil && info.IsDir() {
  242. skip = fs.SkipDir
  243. }
  244. if !utf8.ValidString(path) {
  245. handleError(ctx, "scan", path, errUTF8Invalid, finishedChan)
  246. return skip
  247. }
  248. if fs.IsTemporary(path) {
  249. l.Debugln(w, "temporary:", path, "err:", err)
  250. if err == nil && info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) {
  251. w.Filesystem.Remove(path)
  252. l.Debugln(w, "removing temporary:", path, info.ModTime())
  253. }
  254. return nil
  255. }
  256. if fs.IsInternal(path) {
  257. l.Debugln(w, "ignored (internal):", path)
  258. return skip
  259. }
  260. // Just in case the filesystem doesn't produce the normalization the OS
  261. // uses, and we use internally.
  262. nonNormPath := path
  263. path = normalizePath(path)
  264. if m := w.Matcher.Match(path); m.IsIgnored() {
  265. l.Debugln(w, "ignored (patterns):", path)
  266. // Only descend if matcher says so and the current file is not a symlink.
  267. if err != nil || m.CanSkipDir() || info.IsSymlink() {
  268. return skip
  269. }
  270. // If the parent wasn't ignored already, set this path as the "highest" ignored parent
  271. if info.IsDir() && (ignoredParent == "" || !fs.IsParent(path, ignoredParent)) {
  272. ignoredParent = path
  273. }
  274. return nil
  275. }
  276. if err != nil {
  277. // No need reporting errors for files that don't exist (e.g. scan
  278. // due to filesystem watcher)
  279. if !fs.IsNotExist(err) {
  280. handleError(ctx, "scan", path, err, finishedChan)
  281. }
  282. return skip
  283. }
  284. if path == "." {
  285. return nil
  286. }
  287. if path != nonNormPath {
  288. if !w.AutoNormalize {
  289. // We're not authorized to do anything about it, so complain and skip.
  290. handleError(ctx, "normalizing path", nonNormPath, errUTF8Normalization, finishedChan)
  291. return skip
  292. }
  293. path, err = w.applyNormalization(nonNormPath, path, info)
  294. if err != nil {
  295. handleError(ctx, "normalizing path", nonNormPath, err, finishedChan)
  296. return skip
  297. }
  298. }
  299. if ignoredParent == "" {
  300. // parent isn't ignored, nothing special
  301. if err := w.handleItem(ctx, path, info, toHashChan, finishedChan); err != nil {
  302. handleError(ctx, "scan", path, err, finishedChan)
  303. return skip
  304. }
  305. return nil
  306. }
  307. // Part of current path below the ignored (potential) parent
  308. rel := strings.TrimPrefix(path, ignoredParent+string(fs.PathSeparator))
  309. // ignored path isn't actually a parent of the current path
  310. if rel == path {
  311. ignoredParent = ""
  312. if err := w.handleItem(ctx, path, info, toHashChan, finishedChan); err != nil {
  313. handleError(ctx, "scan", path, err, finishedChan)
  314. return skip
  315. }
  316. return nil
  317. }
  318. // The previously ignored parent directories of the current, not
  319. // ignored path need to be handled as well.
  320. // Prepend an empty string to handle ignoredParent without anything
  321. // appended in the first iteration.
  322. for _, name := range append([]string{""}, fs.PathComponents(rel)...) {
  323. ignoredParent = filepath.Join(ignoredParent, name)
  324. info, err = w.Filesystem.Lstat(ignoredParent)
  325. // An error here would be weird as we've already gotten to this point, but act on it nonetheless
  326. if err != nil {
  327. handleError(ctx, "scan", ignoredParent, err, finishedChan)
  328. return skip
  329. }
  330. if err = w.handleItem(ctx, ignoredParent, info, toHashChan, finishedChan); err != nil {
  331. handleError(ctx, "scan", path, err, finishedChan)
  332. return skip
  333. }
  334. }
  335. ignoredParent = ""
  336. return nil
  337. }
  338. }
  339. // Returning an error does not indicate that the walk should be aborted - it
  340. // will simply report the error for that path to the user (same for walk...
  341. // functions called from here).
  342. func (w *walker) handleItem(ctx context.Context, path string, info fs.FileInfo, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) error {
  343. switch {
  344. case info.IsSymlink():
  345. if err := w.walkSymlink(ctx, path, info, finishedChan); err != nil {
  346. return err
  347. }
  348. if info.IsDir() {
  349. // under no circumstances shall we descend into a symlink
  350. return fs.SkipDir
  351. }
  352. return nil
  353. case info.IsDir():
  354. return w.walkDir(ctx, path, info, finishedChan)
  355. case info.IsRegular():
  356. return w.walkRegular(ctx, path, info, toHashChan)
  357. default:
  358. // A special file, socket, fifo, etc. -- do nothing, just skip and continue scanning.
  359. l.Debugf("Skipping non-regular file %s (%s)", path, info.Mode())
  360. return nil
  361. }
  362. }
  363. func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileInfo, toHashChan chan<- protocol.FileInfo) error {
  364. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  365. blockSize := protocol.BlockSize(info.Size())
  366. if hasCurFile {
  367. // Check if we should retain current block size.
  368. curBlockSize := curFile.BlockSize()
  369. if blockSize > curBlockSize && blockSize/curBlockSize <= 2 {
  370. // New block size is larger, but not more than twice larger.
  371. // Retain.
  372. blockSize = curBlockSize
  373. } else if curBlockSize > blockSize && curBlockSize/blockSize <= 2 {
  374. // Old block size is larger, but not more than twice larger.
  375. // Retain.
  376. blockSize = curBlockSize
  377. }
  378. }
  379. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  380. if err != nil {
  381. return err
  382. }
  383. f = w.updateFileInfo(f, curFile)
  384. f.NoPermissions = w.IgnorePerms
  385. f.RawBlockSize = int32(blockSize)
  386. l.Debugln(w, "checking:", f)
  387. if hasCurFile {
  388. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  389. ModTimeWindow: w.ModTimeWindow,
  390. IgnorePerms: w.IgnorePerms,
  391. IgnoreBlocks: true,
  392. IgnoreFlags: w.LocalFlags,
  393. IgnoreOwnership: !w.ScanOwnership,
  394. IgnoreXattrs: !w.ScanXattrs,
  395. }) {
  396. l.Debugln(w, "unchanged:", curFile)
  397. return nil
  398. }
  399. if curFile.ShouldConflict() && !f.ShouldConflict() {
  400. // The old file was invalid for whatever reason and probably not
  401. // up to date with what was out there in the cluster. Drop all
  402. // others from the version vector to indicate that we haven't
  403. // taken their version into account, and possibly cause a
  404. // conflict. However, only do this if the new file is not also
  405. // invalid. This would indicate that the new file is not part
  406. // of the cluster, but e.g. a local change.
  407. f.Version = f.Version.DropOthers(w.ShortID)
  408. }
  409. l.Debugln(w, "rescan:", curFile)
  410. }
  411. l.Debugln(w, "to hash:", relPath, f)
  412. select {
  413. case toHashChan <- f:
  414. case <-ctx.Done():
  415. return ctx.Err()
  416. }
  417. return nil
  418. }
  419. func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo, finishedChan chan<- ScanResult) error {
  420. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  421. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  422. if err != nil {
  423. return err
  424. }
  425. f = w.updateFileInfo(f, curFile)
  426. f.NoPermissions = w.IgnorePerms
  427. l.Debugln(w, "checking:", f)
  428. if hasCurFile {
  429. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  430. ModTimeWindow: w.ModTimeWindow,
  431. IgnorePerms: w.IgnorePerms,
  432. IgnoreBlocks: true,
  433. IgnoreFlags: w.LocalFlags,
  434. IgnoreOwnership: !w.ScanOwnership,
  435. IgnoreXattrs: !w.ScanXattrs,
  436. }) {
  437. l.Debugln(w, "unchanged:", curFile)
  438. return nil
  439. }
  440. if curFile.ShouldConflict() && !f.ShouldConflict() {
  441. // The old file was invalid for whatever reason and probably not
  442. // up to date with what was out there in the cluster. Drop all
  443. // others from the version vector to indicate that we haven't
  444. // taken their version into account, and possibly cause a
  445. // conflict. However, only do this if the new file is not also
  446. // invalid. This would indicate that the new file is not part
  447. // of the cluster, but e.g. a local change.
  448. f.Version = f.Version.DropOthers(w.ShortID)
  449. }
  450. l.Debugln(w, "rescan:", curFile)
  451. }
  452. l.Debugln(w, "dir:", relPath, f)
  453. select {
  454. case finishedChan <- ScanResult{File: f}:
  455. case <-ctx.Done():
  456. return ctx.Err()
  457. }
  458. return nil
  459. }
  460. func (w *walker) walkSymlink(ctx context.Context, relPath string, info fs.FileInfo, finishedChan chan<- ScanResult) error {
  461. // Symlinks are not supported on Windows. We ignore instead of returning
  462. // an error.
  463. if build.IsWindows {
  464. return nil
  465. }
  466. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  467. if err != nil {
  468. return err
  469. }
  470. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  471. f = w.updateFileInfo(f, curFile)
  472. l.Debugln(w, "checking:", f)
  473. if hasCurFile {
  474. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  475. ModTimeWindow: w.ModTimeWindow,
  476. IgnorePerms: w.IgnorePerms,
  477. IgnoreBlocks: true,
  478. IgnoreFlags: w.LocalFlags,
  479. IgnoreOwnership: !w.ScanOwnership,
  480. IgnoreXattrs: !w.ScanXattrs,
  481. }) {
  482. l.Debugln(w, "unchanged:", curFile, info.ModTime().Unix(), info.Mode()&fs.ModePerm)
  483. return nil
  484. }
  485. if curFile.ShouldConflict() && !f.ShouldConflict() {
  486. // The old file was invalid for whatever reason and probably not
  487. // up to date with what was out there in the cluster. Drop all
  488. // others from the version vector to indicate that we haven't
  489. // taken their version into account, and possibly cause a
  490. // conflict. However, only do this if the new file is not also
  491. // invalid. This would indicate that the new file is not part
  492. // of the cluster, but e.g. a local change.
  493. f.Version = f.Version.DropOthers(w.ShortID)
  494. }
  495. l.Debugln(w, "rescan:", curFile)
  496. }
  497. l.Debugln(w, "symlink:", relPath, f)
  498. select {
  499. case finishedChan <- ScanResult{File: f}:
  500. case <-ctx.Done():
  501. return ctx.Err()
  502. }
  503. return nil
  504. }
  505. func normalizePath(path string) string {
  506. if build.IsDarwin || build.IsIOS {
  507. // Mac OS X file names should always be NFD normalized.
  508. return norm.NFD.String(path)
  509. }
  510. // Every other OS in the known universe uses NFC or just plain
  511. // doesn't bother to define an encoding. In our case *we* do care,
  512. // so we enforce NFC regardless.
  513. return norm.NFC.String(path)
  514. }
  515. // applyNormalization fixes the normalization of the file on disk, i.e. ensures
  516. // the file at path ends up named normPath. It shouldn't but may happen that the
  517. // file ends up with a different name, in which case that one should be scanned.
  518. func (w *walker) applyNormalization(path, normPath string, info fs.FileInfo) (string, error) {
  519. // We will attempt to normalize it.
  520. normInfo, err := w.Filesystem.Lstat(normPath)
  521. if fs.IsNotExist(err) {
  522. // Nothing exists with the normalized filename. Good.
  523. if err = w.Filesystem.Rename(path, normPath); err != nil {
  524. return "", err
  525. }
  526. l.Infof(`Normalized UTF8 encoding of file name "%s".`, path)
  527. return normPath, nil
  528. }
  529. if w.Filesystem.SameFile(info, normInfo) {
  530. // With some filesystems (ZFS), if there is an un-normalized path and you ask whether the normalized
  531. // version exists, it responds with true. Therefore we need to check fs.SameFile as well.
  532. // In this case, a call to Rename won't do anything, so we have to rename via a temp file.
  533. // We don't want to use the standard syncthing prefix here, as that will result in the file being ignored
  534. // and eventually deleted by Syncthing if the rename back fails.
  535. tempPath := fs.TempNameWithPrefix(normPath, "")
  536. if err = w.Filesystem.Rename(path, tempPath); err != nil {
  537. return "", err
  538. }
  539. if err = w.Filesystem.Rename(tempPath, normPath); err != nil {
  540. // I don't ever expect this to happen, but if it does, we should probably tell our caller that the normalized
  541. // path is the temp path: that way at least the user's data still gets synced.
  542. l.Warnf(`Error renaming "%s" to "%s" while normalizating UTF8 encoding: %v. You will want to rename this file back manually`, tempPath, normPath, err)
  543. return tempPath, nil
  544. }
  545. return normPath, nil
  546. }
  547. // There is something already in the way at the normalized
  548. // file name.
  549. return "", errUTF8Conflict
  550. }
  551. // updateFileInfo updates walker specific members of protocol.FileInfo that
  552. // do not depend on type, and things that should be preserved from the
  553. // previous version of the FileInfo.
  554. func (w *walker) updateFileInfo(dst, src protocol.FileInfo) protocol.FileInfo {
  555. if dst.Type == protocol.FileInfoTypeFile && build.IsWindows {
  556. // If we have an existing index entry, copy the executable bits
  557. // from there.
  558. dst.Permissions |= (src.Permissions & 0o111)
  559. }
  560. dst.Version = src.Version.Update(w.ShortID)
  561. dst.ModifiedBy = w.ShortID
  562. dst.LocalFlags = w.LocalFlags
  563. // Copy OS data from src to dst, unless it was already set on dst.
  564. dst.Platform.MergeWith(&src.Platform)
  565. return dst
  566. }
  567. func handleError(ctx context.Context, context, path string, err error, finishedChan chan<- ScanResult) {
  568. l.Debugf("handle error on '%v': %v: %v", path, context, err)
  569. select {
  570. case finishedChan <- ScanResult{
  571. Err: fmt.Errorf("%s: %w", context, err),
  572. Path: path,
  573. }:
  574. case <-ctx.Done():
  575. }
  576. }
  577. func (w *walker) String() string {
  578. return fmt.Sprintf("walker/%s@%p", w.Folder, w)
  579. }
  580. // A byteCounter gets bytes added to it via Update() and then provides the
  581. // Total() and one minute moving average Rate() in bytes per second.
  582. type byteCounter struct {
  583. total atomic.Int64
  584. metrics.EWMA
  585. stop chan struct{}
  586. }
  587. func newByteCounter() *byteCounter {
  588. c := &byteCounter{
  589. EWMA: metrics.NewEWMA1(), // a one minute exponentially weighted moving average
  590. stop: make(chan struct{}),
  591. }
  592. go c.ticker()
  593. return c
  594. }
  595. func (c *byteCounter) ticker() {
  596. // The metrics.EWMA expects clock ticks every five seconds in order to
  597. // decay the average properly.
  598. t := time.NewTicker(5 * time.Second)
  599. for {
  600. select {
  601. case <-t.C:
  602. c.Tick()
  603. case <-c.stop:
  604. t.Stop()
  605. return
  606. }
  607. }
  608. }
  609. func (c *byteCounter) Update(bytes int64) {
  610. c.total.Add(bytes)
  611. c.EWMA.Update(bytes)
  612. }
  613. func (c *byteCounter) Total() int64 { return c.total.Load() }
  614. func (c *byteCounter) Close() {
  615. close(c.stop)
  616. }
  617. // A no-op CurrentFiler
  618. type noCurrentFiler struct{}
  619. func (noCurrentFiler) CurrentFile(_ string) (protocol.FileInfo, bool) {
  620. return protocol.FileInfo{}, false
  621. }
  622. func CreateFileInfo(fi fs.FileInfo, name string, filesystem fs.Filesystem, scanOwnership bool, scanXattrs bool, xattrFilter XattrFilter) (protocol.FileInfo, error) {
  623. f := protocol.FileInfo{Name: name}
  624. if scanOwnership || scanXattrs {
  625. if plat, err := filesystem.PlatformData(name, scanOwnership, scanXattrs, xattrFilter); err == nil {
  626. f.Platform = plat
  627. } else {
  628. return protocol.FileInfo{}, fmt.Errorf("reading platform data: %w", err)
  629. }
  630. }
  631. if ct := fi.InodeChangeTime(); !ct.IsZero() {
  632. f.InodeChangeNs = ct.UnixNano()
  633. } else {
  634. f.InodeChangeNs = 0
  635. }
  636. if fi.IsSymlink() {
  637. f.Type = protocol.FileInfoTypeSymlink
  638. target, err := filesystem.ReadSymlink(name)
  639. if err != nil {
  640. return protocol.FileInfo{}, err
  641. }
  642. f.SymlinkTarget = []byte(target)
  643. f.NoPermissions = true // Symlinks don't have permissions of their own
  644. return f, nil
  645. }
  646. f.Permissions = uint32(fi.Mode() & fs.ModePerm)
  647. f.ModifiedS = fi.ModTime().Unix()
  648. f.ModifiedNs = int32(fi.ModTime().Nanosecond())
  649. if fi.IsDir() {
  650. f.Type = protocol.FileInfoTypeDirectory
  651. return f, nil
  652. }
  653. f.Size = fi.Size()
  654. f.Type = protocol.FileInfoTypeFile
  655. return f, nil
  656. }