walk.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769
  1. // Copyright (C) 2014 The Syncthing Authors.
  2. //
  3. // This Source Code Form is subject to the terms of the Mozilla Public
  4. // License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5. // You can obtain one at https://mozilla.org/MPL/2.0/.
  6. package scanner
  7. import (
  8. "context"
  9. "errors"
  10. "fmt"
  11. "path/filepath"
  12. "strings"
  13. "sync/atomic"
  14. "time"
  15. "unicode/utf8"
  16. metrics "github.com/rcrowley/go-metrics"
  17. "golang.org/x/text/unicode/norm"
  18. "github.com/syncthing/syncthing/lib/build"
  19. "github.com/syncthing/syncthing/lib/events"
  20. "github.com/syncthing/syncthing/lib/fs"
  21. "github.com/syncthing/syncthing/lib/ignore"
  22. "github.com/syncthing/syncthing/lib/osutil"
  23. "github.com/syncthing/syncthing/lib/protocol"
  24. )
  25. type Config struct {
  26. // Folder for which the walker has been created
  27. Folder string
  28. // Limit walking to these paths within Dir, or no limit if Sub is empty
  29. Subs []string
  30. // If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
  31. Matcher *ignore.Matcher
  32. // Number of hours to keep temporary files for
  33. TempLifetime time.Duration
  34. // If CurrentFiler is not nil, it is queried for the current file before rescanning.
  35. CurrentFiler CurrentFiler
  36. // The Filesystem provides an abstraction on top of the actual filesystem.
  37. Filesystem fs.Filesystem
  38. // If IgnorePerms is true, changes to permission bits will not be
  39. // detected.
  40. IgnorePerms bool
  41. // When AutoNormalize is set, file names that are in UTF8 but incorrect
  42. // normalization form will be corrected.
  43. AutoNormalize bool
  44. // Number of routines to use for hashing
  45. Hashers int
  46. // Our vector clock id
  47. ShortID protocol.ShortID
  48. // Optional progress tick interval which defines how often FolderScanProgress
  49. // events are emitted. Negative number means disabled.
  50. ProgressTickIntervalS int
  51. // Local flags to set on scanned files
  52. LocalFlags uint32
  53. // Modification time is to be considered unchanged if the difference is lower.
  54. ModTimeWindow time.Duration
  55. // Event logger to which the scan progress events are sent
  56. EventLogger events.Logger
  57. // If ScanOwnership is true, we pick up ownership information on files while scanning.
  58. ScanOwnership bool
  59. // If ScanXattrs is true, we pick up extended attributes on files while scanning.
  60. ScanXattrs bool
  61. // Filter for extended attributes
  62. XattrFilter XattrFilter
  63. }
  64. type CurrentFiler interface {
  65. // CurrentFile returns the file as seen at last scan.
  66. CurrentFile(name string) (protocol.FileInfo, bool)
  67. }
  68. type XattrFilter interface {
  69. Permit(string) bool
  70. GetMaxSingleEntrySize() int
  71. GetMaxTotalSize() int
  72. }
  73. type ScanResult struct {
  74. File protocol.FileInfo
  75. Err error
  76. Path string // to be set in case Err != nil and File == nil
  77. }
  78. func Walk(ctx context.Context, cfg Config) chan ScanResult {
  79. return newWalker(cfg).walk(ctx)
  80. }
  81. func WalkWithoutHashing(ctx context.Context, cfg Config) chan ScanResult {
  82. return newWalker(cfg).walkWithoutHashing(ctx)
  83. }
  84. func newWalker(cfg Config) *walker {
  85. w := &walker{cfg}
  86. if w.CurrentFiler == nil {
  87. w.CurrentFiler = noCurrentFiler{}
  88. }
  89. if w.Filesystem == nil {
  90. panic("no filesystem specified")
  91. }
  92. if w.Matcher == nil {
  93. w.Matcher = ignore.New(w.Filesystem)
  94. }
  95. registerFolderMetrics(w.Folder)
  96. return w
  97. }
  98. var (
  99. errUTF8Invalid = errors.New("item is not in UTF8 encoding")
  100. errUTF8Normalization = errors.New("item is not in the correct UTF8 normalization form")
  101. errUTF8Conflict = errors.New("item has UTF8 encoding conflict with another item")
  102. )
  103. type walker struct {
  104. Config
  105. }
  106. // Walk returns the list of files found in the local folder by scanning the
  107. // file system. Files are blockwise hashed.
  108. func (w *walker) walk(ctx context.Context) chan ScanResult {
  109. l.Debugln(w, "Walk", w.Subs, w.Matcher)
  110. toHashChan := make(chan protocol.FileInfo)
  111. finishedChan := make(chan ScanResult)
  112. // A routine which walks the filesystem tree, and sends files which have
  113. // been modified to the counter routine.
  114. go w.scan(ctx, toHashChan, finishedChan)
  115. // We're not required to emit scan progress events, just kick off hashers,
  116. // and feed inputs directly from the walker.
  117. if w.ProgressTickIntervalS < 0 {
  118. newParallelHasher(ctx, w.Folder, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil)
  119. return finishedChan
  120. }
  121. // Defaults to every 2 seconds.
  122. if w.ProgressTickIntervalS == 0 {
  123. w.ProgressTickIntervalS = 2
  124. }
  125. // We need to emit progress events, hence we create a routine which buffers
  126. // the list of files to be hashed, counts the total number of
  127. // bytes to hash, and once no more files need to be hashed (chan gets closed),
  128. // start a routine which periodically emits FolderScanProgress events,
  129. // until a stop signal is sent by the parallel hasher.
  130. // Parallel hasher is stopped by this routine when we close the channel over
  131. // which it receives the files we ask it to hash.
  132. go func() {
  133. var filesToHash []protocol.FileInfo
  134. var total int64 = 1
  135. for file := range toHashChan {
  136. filesToHash = append(filesToHash, file)
  137. total += file.Size
  138. }
  139. if len(filesToHash) == 0 {
  140. close(finishedChan)
  141. return
  142. }
  143. realToHashChan := make(chan protocol.FileInfo)
  144. done := make(chan struct{})
  145. progress := newByteCounter()
  146. newParallelHasher(ctx, w.Folder, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done)
  147. // A routine which actually emits the FolderScanProgress events
  148. // every w.ProgressTicker ticks, until the hasher routines terminate.
  149. go func() {
  150. defer progress.Close()
  151. emitProgressEvent := func() {
  152. current := progress.Total()
  153. rate := progress.Rate()
  154. l.Debugf("%v: Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w, w.Folder, w.Subs, current, total, rate/1024/1024, current*100/total)
  155. w.EventLogger.Log(events.FolderScanProgress, map[string]interface{}{
  156. "folder": w.Folder,
  157. "current": current,
  158. "total": total,
  159. "rate": rate, // bytes per second
  160. })
  161. }
  162. ticker := time.NewTicker(time.Duration(w.ProgressTickIntervalS) * time.Second)
  163. defer ticker.Stop()
  164. for {
  165. select {
  166. case <-done:
  167. emitProgressEvent()
  168. l.Debugln(w, "Walk progress done", w.Folder, w.Subs, w.Matcher)
  169. return
  170. case <-ticker.C:
  171. emitProgressEvent()
  172. case <-ctx.Done():
  173. return
  174. }
  175. }
  176. }()
  177. loop:
  178. for _, file := range filesToHash {
  179. l.Debugln(w, "real to hash:", file.Name)
  180. select {
  181. case realToHashChan <- file:
  182. case <-ctx.Done():
  183. break loop
  184. }
  185. }
  186. close(realToHashChan)
  187. }()
  188. return finishedChan
  189. }
  190. func (w *walker) walkWithoutHashing(ctx context.Context) chan ScanResult {
  191. l.Debugln(w, "Walk without hashing", w.Subs, w.Matcher)
  192. toHashChan := make(chan protocol.FileInfo)
  193. finishedChan := make(chan ScanResult)
  194. // A routine which walks the filesystem tree, and sends files which have
  195. // been modified to the counter routine.
  196. go w.scan(ctx, toHashChan, finishedChan)
  197. go func() {
  198. for file := range toHashChan {
  199. finishedChan <- ScanResult{File: file}
  200. }
  201. close(finishedChan)
  202. }()
  203. return finishedChan
  204. }
  205. const walkFailureEventDesc = "Unexpected error while walking the filesystem during scan"
  206. func (w *walker) scan(ctx context.Context, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) {
  207. hashFiles := w.walkAndHashFiles(ctx, toHashChan, finishedChan)
  208. if len(w.Subs) == 0 {
  209. if err := w.Filesystem.Walk(".", hashFiles); isWarnableError(err) {
  210. w.EventLogger.Log(events.Failure, walkFailureEventDesc)
  211. l.Warnf("Aborted scan due to an unexpected error: %v", err)
  212. }
  213. } else {
  214. for _, sub := range w.Subs {
  215. if err := osutil.TraversesSymlink(w.Filesystem, filepath.Dir(sub)); err != nil {
  216. l.Debugf("%v: Skip walking %v as it is below a symlink", w, sub)
  217. continue
  218. }
  219. if err := w.Filesystem.Walk(sub, hashFiles); isWarnableError(err) {
  220. w.EventLogger.Log(events.Failure, walkFailureEventDesc)
  221. l.Warnf("Aborted scan of path '%v' due to an unexpected error: %v", sub, err)
  222. }
  223. }
  224. }
  225. close(toHashChan)
  226. }
  227. // isWarnableError returns true if err is a kind of error we should warn
  228. // about receiving from the folder walk.
  229. func isWarnableError(err error) bool {
  230. return err != nil &&
  231. !errors.Is(err, fs.SkipDir) && // intentional skip
  232. !errors.Is(err, context.Canceled) // folder restarting
  233. }
  234. func (w *walker) walkAndHashFiles(ctx context.Context, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) fs.WalkFunc {
  235. now := time.Now()
  236. ignoredParent := ""
  237. return func(path string, info fs.FileInfo, err error) error {
  238. select {
  239. case <-ctx.Done():
  240. return ctx.Err()
  241. default:
  242. }
  243. metricScannedItems.WithLabelValues(w.Folder).Inc()
  244. // Return value used when we are returning early and don't want to
  245. // process the item. For directories, this means do-not-descend.
  246. var skip error // nil
  247. // info nil when error is not nil
  248. if info != nil && info.IsDir() {
  249. skip = fs.SkipDir
  250. }
  251. if !utf8.ValidString(path) {
  252. handleError(ctx, "scan", path, errUTF8Invalid, finishedChan)
  253. return skip
  254. }
  255. if fs.IsTemporary(path) {
  256. l.Debugln(w, "temporary:", path, "err:", err)
  257. if err == nil && info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) {
  258. w.Filesystem.Remove(path)
  259. l.Debugln(w, "removing temporary:", path, info.ModTime())
  260. }
  261. return nil
  262. }
  263. if fs.IsInternal(path) {
  264. l.Debugln(w, "ignored (internal):", path)
  265. return skip
  266. }
  267. // Just in case the filesystem doesn't produce the normalization the OS
  268. // uses, and we use internally.
  269. nonNormPath := path
  270. path = normalizePath(path)
  271. if m := w.Matcher.Match(path); m.IsIgnored() {
  272. l.Debugln(w, "ignored (patterns):", path)
  273. // Only descend if matcher says so and the current file is not a symlink.
  274. if err != nil || m.CanSkipDir() || info.IsSymlink() {
  275. return skip
  276. }
  277. // If the parent wasn't ignored already, set this path as the "highest" ignored parent
  278. if info.IsDir() && (ignoredParent == "" || !fs.IsParent(path, ignoredParent)) {
  279. ignoredParent = path
  280. }
  281. return nil
  282. }
  283. if err != nil {
  284. // No need reporting errors for files that don't exist (e.g. scan
  285. // due to filesystem watcher)
  286. if !fs.IsNotExist(err) {
  287. handleError(ctx, "scan", path, err, finishedChan)
  288. }
  289. return skip
  290. }
  291. if path == "." {
  292. return nil
  293. }
  294. if path != nonNormPath {
  295. if !w.AutoNormalize {
  296. // We're not authorized to do anything about it, so complain and skip.
  297. handleError(ctx, "normalizing path", nonNormPath, errUTF8Normalization, finishedChan)
  298. return skip
  299. }
  300. path, err = w.applyNormalization(nonNormPath, path, info)
  301. if err != nil {
  302. handleError(ctx, "normalizing path", nonNormPath, err, finishedChan)
  303. return skip
  304. }
  305. }
  306. if ignoredParent == "" {
  307. // parent isn't ignored, nothing special
  308. if err := w.handleItem(ctx, path, info, toHashChan, finishedChan); err != nil {
  309. handleError(ctx, "scan", path, err, finishedChan)
  310. return skip
  311. }
  312. return nil
  313. }
  314. // Part of current path below the ignored (potential) parent
  315. rel := strings.TrimPrefix(path, ignoredParent+string(fs.PathSeparator))
  316. // ignored path isn't actually a parent of the current path
  317. if rel == path {
  318. ignoredParent = ""
  319. if err := w.handleItem(ctx, path, info, toHashChan, finishedChan); err != nil {
  320. handleError(ctx, "scan", path, err, finishedChan)
  321. return skip
  322. }
  323. return nil
  324. }
  325. // The previously ignored parent directories of the current, not
  326. // ignored path need to be handled as well.
  327. // Prepend an empty string to handle ignoredParent without anything
  328. // appended in the first iteration.
  329. for _, name := range append([]string{""}, fs.PathComponents(rel)...) {
  330. ignoredParent = filepath.Join(ignoredParent, name)
  331. info, err = w.Filesystem.Lstat(ignoredParent)
  332. // An error here would be weird as we've already gotten to this point, but act on it nonetheless
  333. if err != nil {
  334. handleError(ctx, "scan", ignoredParent, err, finishedChan)
  335. return skip
  336. }
  337. if err = w.handleItem(ctx, ignoredParent, info, toHashChan, finishedChan); err != nil {
  338. handleError(ctx, "scan", path, err, finishedChan)
  339. return skip
  340. }
  341. }
  342. ignoredParent = ""
  343. return nil
  344. }
  345. }
  346. // Returning an error does not indicate that the walk should be aborted - it
  347. // will simply report the error for that path to the user (same for walk...
  348. // functions called from here).
  349. func (w *walker) handleItem(ctx context.Context, path string, info fs.FileInfo, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) error {
  350. switch {
  351. case info.IsSymlink():
  352. if err := w.walkSymlink(ctx, path, info, finishedChan); err != nil {
  353. return err
  354. }
  355. if info.IsDir() {
  356. // under no circumstances shall we descend into a symlink
  357. return fs.SkipDir
  358. }
  359. return nil
  360. case info.IsDir():
  361. return w.walkDir(ctx, path, info, finishedChan)
  362. case info.IsRegular():
  363. return w.walkRegular(ctx, path, info, toHashChan)
  364. default:
  365. // A special file, socket, fifo, etc. -- do nothing, just skip and continue scanning.
  366. l.Debugf("Skipping non-regular file %s (%s)", path, info.Mode())
  367. return nil
  368. }
  369. }
  370. func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileInfo, toHashChan chan<- protocol.FileInfo) error {
  371. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  372. blockSize := protocol.BlockSize(info.Size())
  373. if hasCurFile {
  374. // Check if we should retain current block size.
  375. curBlockSize := curFile.BlockSize()
  376. if blockSize > curBlockSize && blockSize/curBlockSize <= 2 {
  377. // New block size is larger, but not more than twice larger.
  378. // Retain.
  379. blockSize = curBlockSize
  380. } else if curBlockSize > blockSize && curBlockSize/blockSize <= 2 {
  381. // Old block size is larger, but not more than twice larger.
  382. // Retain.
  383. blockSize = curBlockSize
  384. }
  385. }
  386. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  387. if err != nil {
  388. return err
  389. }
  390. f = w.updateFileInfo(f, curFile)
  391. f.NoPermissions = w.IgnorePerms
  392. f.RawBlockSize = int32(blockSize)
  393. l.Debugln(w, "checking:", f)
  394. if hasCurFile {
  395. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  396. ModTimeWindow: w.ModTimeWindow,
  397. IgnorePerms: w.IgnorePerms,
  398. IgnoreBlocks: true,
  399. IgnoreFlags: w.LocalFlags,
  400. IgnoreOwnership: !w.ScanOwnership,
  401. IgnoreXattrs: !w.ScanXattrs,
  402. }) {
  403. l.Debugln(w, "unchanged:", curFile)
  404. return nil
  405. }
  406. if curFile.ShouldConflict() && !f.ShouldConflict() {
  407. // The old file was invalid for whatever reason and probably not
  408. // up to date with what was out there in the cluster. Drop all
  409. // others from the version vector to indicate that we haven't
  410. // taken their version into account, and possibly cause a
  411. // conflict. However, only do this if the new file is not also
  412. // invalid. This would indicate that the new file is not part
  413. // of the cluster, but e.g. a local change.
  414. f.Version = f.Version.DropOthers(w.ShortID)
  415. }
  416. l.Debugln(w, "rescan:", curFile)
  417. }
  418. l.Debugln(w, "to hash:", relPath, f)
  419. select {
  420. case toHashChan <- f:
  421. case <-ctx.Done():
  422. return ctx.Err()
  423. }
  424. return nil
  425. }
  426. func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo, finishedChan chan<- ScanResult) error {
  427. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  428. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  429. if err != nil {
  430. return err
  431. }
  432. f = w.updateFileInfo(f, curFile)
  433. f.NoPermissions = w.IgnorePerms
  434. l.Debugln(w, "checking:", f)
  435. if hasCurFile {
  436. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  437. ModTimeWindow: w.ModTimeWindow,
  438. IgnorePerms: w.IgnorePerms,
  439. IgnoreBlocks: true,
  440. IgnoreFlags: w.LocalFlags,
  441. IgnoreOwnership: !w.ScanOwnership,
  442. IgnoreXattrs: !w.ScanXattrs,
  443. }) {
  444. l.Debugln(w, "unchanged:", curFile)
  445. return nil
  446. }
  447. if curFile.ShouldConflict() && !f.ShouldConflict() {
  448. // The old file was invalid for whatever reason and probably not
  449. // up to date with what was out there in the cluster. Drop all
  450. // others from the version vector to indicate that we haven't
  451. // taken their version into account, and possibly cause a
  452. // conflict. However, only do this if the new file is not also
  453. // invalid. This would indicate that the new file is not part
  454. // of the cluster, but e.g. a local change.
  455. f.Version = f.Version.DropOthers(w.ShortID)
  456. }
  457. l.Debugln(w, "rescan:", curFile)
  458. }
  459. l.Debugln(w, "dir:", relPath, f)
  460. select {
  461. case finishedChan <- ScanResult{File: f}:
  462. case <-ctx.Done():
  463. return ctx.Err()
  464. }
  465. return nil
  466. }
  467. func (w *walker) walkSymlink(ctx context.Context, relPath string, info fs.FileInfo, finishedChan chan<- ScanResult) error {
  468. // Symlinks are not supported on Windows. We ignore instead of returning
  469. // an error.
  470. if build.IsWindows {
  471. return nil
  472. }
  473. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  474. if err != nil {
  475. return err
  476. }
  477. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  478. f = w.updateFileInfo(f, curFile)
  479. l.Debugln(w, "checking:", f)
  480. if hasCurFile {
  481. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  482. ModTimeWindow: w.ModTimeWindow,
  483. IgnorePerms: w.IgnorePerms,
  484. IgnoreBlocks: true,
  485. IgnoreFlags: w.LocalFlags,
  486. IgnoreOwnership: !w.ScanOwnership,
  487. IgnoreXattrs: !w.ScanXattrs,
  488. }) {
  489. l.Debugln(w, "unchanged:", curFile, info.ModTime().Unix(), info.Mode()&fs.ModePerm)
  490. return nil
  491. }
  492. if curFile.ShouldConflict() && !f.ShouldConflict() {
  493. // The old file was invalid for whatever reason and probably not
  494. // up to date with what was out there in the cluster. Drop all
  495. // others from the version vector to indicate that we haven't
  496. // taken their version into account, and possibly cause a
  497. // conflict. However, only do this if the new file is not also
  498. // invalid. This would indicate that the new file is not part
  499. // of the cluster, but e.g. a local change.
  500. f.Version = f.Version.DropOthers(w.ShortID)
  501. }
  502. l.Debugln(w, "rescan:", curFile)
  503. }
  504. l.Debugln(w, "symlink:", relPath, f)
  505. select {
  506. case finishedChan <- ScanResult{File: f}:
  507. case <-ctx.Done():
  508. return ctx.Err()
  509. }
  510. return nil
  511. }
  512. func normalizePath(path string) string {
  513. if build.IsDarwin || build.IsIOS {
  514. // Mac OS X file names should always be NFD normalized.
  515. return norm.NFD.String(path)
  516. }
  517. // Every other OS in the known universe uses NFC or just plain
  518. // doesn't bother to define an encoding. In our case *we* do care,
  519. // so we enforce NFC regardless.
  520. return norm.NFC.String(path)
  521. }
  522. // applyNormalization fixes the normalization of the file on disk, i.e. ensures
  523. // the file at path ends up named normPath. It shouldn't but may happen that the
  524. // file ends up with a different name, in which case that one should be scanned.
  525. func (w *walker) applyNormalization(path, normPath string, info fs.FileInfo) (string, error) {
  526. // We will attempt to normalize it.
  527. normInfo, err := w.Filesystem.Lstat(normPath)
  528. if fs.IsNotExist(err) {
  529. // Nothing exists with the normalized filename. Good.
  530. if err = w.Filesystem.Rename(path, normPath); err != nil {
  531. return "", err
  532. }
  533. l.Infof(`Normalized UTF8 encoding of file name "%s".`, path)
  534. return normPath, nil
  535. }
  536. if w.Filesystem.SameFile(info, normInfo) {
  537. // With some filesystems (ZFS), if there is an un-normalized path and you ask whether the normalized
  538. // version exists, it responds with true. Therefore we need to check fs.SameFile as well.
  539. // In this case, a call to Rename won't do anything, so we have to rename via a temp file.
  540. // We don't want to use the standard syncthing prefix here, as that will result in the file being ignored
  541. // and eventually deleted by Syncthing if the rename back fails.
  542. tempPath := fs.TempNameWithPrefix(normPath, "")
  543. if err = w.Filesystem.Rename(path, tempPath); err != nil {
  544. return "", err
  545. }
  546. if err = w.Filesystem.Rename(tempPath, normPath); err != nil {
  547. // I don't ever expect this to happen, but if it does, we should probably tell our caller that the normalized
  548. // path is the temp path: that way at least the user's data still gets synced.
  549. l.Warnf(`Error renaming "%s" to "%s" while normalizating UTF8 encoding: %v. You will want to rename this file back manually`, tempPath, normPath, err)
  550. return tempPath, nil
  551. }
  552. return normPath, nil
  553. }
  554. // There is something already in the way at the normalized
  555. // file name.
  556. return "", errUTF8Conflict
  557. }
  558. // updateFileInfo updates walker specific members of protocol.FileInfo that
  559. // do not depend on type, and things that should be preserved from the
  560. // previous version of the FileInfo.
  561. func (w *walker) updateFileInfo(dst, src protocol.FileInfo) protocol.FileInfo {
  562. if dst.Type == protocol.FileInfoTypeFile && build.IsWindows {
  563. // If we have an existing index entry, copy the executable bits
  564. // from there.
  565. dst.Permissions |= (src.Permissions & 0o111)
  566. }
  567. dst.Version = src.Version.Update(w.ShortID)
  568. dst.ModifiedBy = w.ShortID
  569. dst.LocalFlags = w.LocalFlags
  570. // Copy OS data from src to dst, unless it was already set on dst.
  571. dst.Platform.MergeWith(&src.Platform)
  572. return dst
  573. }
  574. func handleError(ctx context.Context, context, path string, err error, finishedChan chan<- ScanResult) {
  575. l.Debugf("handle error on '%v': %v: %v", path, context, err)
  576. select {
  577. case finishedChan <- ScanResult{
  578. Err: fmt.Errorf("%s: %w", context, err),
  579. Path: path,
  580. }:
  581. case <-ctx.Done():
  582. }
  583. }
  584. func (w *walker) String() string {
  585. return fmt.Sprintf("walker/%s@%p", w.Folder, w)
  586. }
  587. // A byteCounter gets bytes added to it via Update() and then provides the
  588. // Total() and one minute moving average Rate() in bytes per second.
  589. type byteCounter struct {
  590. total atomic.Int64
  591. metrics.EWMA
  592. stop chan struct{}
  593. }
  594. func newByteCounter() *byteCounter {
  595. c := &byteCounter{
  596. EWMA: metrics.NewEWMA1(), // a one minute exponentially weighted moving average
  597. stop: make(chan struct{}),
  598. }
  599. go c.ticker()
  600. return c
  601. }
  602. func (c *byteCounter) ticker() {
  603. // The metrics.EWMA expects clock ticks every five seconds in order to
  604. // decay the average properly.
  605. t := time.NewTicker(5 * time.Second)
  606. for {
  607. select {
  608. case <-t.C:
  609. c.Tick()
  610. case <-c.stop:
  611. t.Stop()
  612. return
  613. }
  614. }
  615. }
  616. func (c *byteCounter) Update(bytes int64) {
  617. c.total.Add(bytes)
  618. c.EWMA.Update(bytes)
  619. }
  620. func (c *byteCounter) Total() int64 { return c.total.Load() }
  621. func (c *byteCounter) Close() {
  622. close(c.stop)
  623. }
  624. // A no-op CurrentFiler
  625. type noCurrentFiler struct{}
  626. func (noCurrentFiler) CurrentFile(_ string) (protocol.FileInfo, bool) {
  627. return protocol.FileInfo{}, false
  628. }
  629. func CreateFileInfo(fi fs.FileInfo, name string, filesystem fs.Filesystem, scanOwnership bool, scanXattrs bool, xattrFilter XattrFilter) (protocol.FileInfo, error) {
  630. f := protocol.FileInfo{Name: name}
  631. if scanOwnership || scanXattrs {
  632. if plat, err := filesystem.PlatformData(name, scanOwnership, scanXattrs, xattrFilter); err == nil {
  633. f.Platform = plat
  634. } else {
  635. return protocol.FileInfo{}, fmt.Errorf("reading platform data: %w", err)
  636. }
  637. }
  638. if ct := fi.InodeChangeTime(); !ct.IsZero() {
  639. f.InodeChangeNs = ct.UnixNano()
  640. } else {
  641. f.InodeChangeNs = 0
  642. }
  643. if fi.IsSymlink() {
  644. f.Type = protocol.FileInfoTypeSymlink
  645. target, err := filesystem.ReadSymlink(name)
  646. if err != nil {
  647. return protocol.FileInfo{}, err
  648. }
  649. f.SymlinkTarget = []byte(target)
  650. f.NoPermissions = true // Symlinks don't have permissions of their own
  651. return f, nil
  652. }
  653. f.Permissions = uint32(fi.Mode() & fs.ModePerm)
  654. f.ModifiedS = fi.ModTime().Unix()
  655. f.ModifiedNs = int32(fi.ModTime().Nanosecond())
  656. if fi.IsDir() {
  657. f.Type = protocol.FileInfoTypeDirectory
  658. return f, nil
  659. }
  660. f.Size = fi.Size()
  661. f.Type = protocol.FileInfoTypeFile
  662. return f, nil
  663. }