walk.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772
  1. // Copyright (C) 2014 The Syncthing Authors.
  2. //
  3. // This Source Code Form is subject to the terms of the Mozilla Public
  4. // License, v. 2.0. If a copy of the MPL was not distributed with this file,
  5. // You can obtain one at https://mozilla.org/MPL/2.0/.
  6. package scanner
  7. import (
  8. "context"
  9. "errors"
  10. "fmt"
  11. "log/slog"
  12. "path/filepath"
  13. "strings"
  14. "sync/atomic"
  15. "time"
  16. "unicode/utf8"
  17. metrics "github.com/rcrowley/go-metrics"
  18. "golang.org/x/text/unicode/norm"
  19. "github.com/syncthing/syncthing/internal/slogutil"
  20. "github.com/syncthing/syncthing/lib/build"
  21. "github.com/syncthing/syncthing/lib/events"
  22. "github.com/syncthing/syncthing/lib/fs"
  23. "github.com/syncthing/syncthing/lib/ignore"
  24. "github.com/syncthing/syncthing/lib/osutil"
  25. "github.com/syncthing/syncthing/lib/protocol"
  26. )
  27. type Config struct {
  28. // Folder for which the walker has been created
  29. Folder string
  30. // Limit walking to these paths within Dir, or no limit if Sub is empty
  31. Subs []string
  32. // If Matcher is not nil, it is used to identify files to ignore which were specified by the user.
  33. Matcher *ignore.Matcher
  34. // Number of hours to keep temporary files for
  35. TempLifetime time.Duration
  36. // If CurrentFiler is not nil, it is queried for the current file before rescanning.
  37. CurrentFiler CurrentFiler
  38. // The Filesystem provides an abstraction on top of the actual filesystem.
  39. Filesystem fs.Filesystem
  40. // If IgnorePerms is true, changes to permission bits will not be
  41. // detected.
  42. IgnorePerms bool
  43. // When AutoNormalize is set, file names that are in UTF8 but incorrect
  44. // normalization form will be corrected.
  45. AutoNormalize bool
  46. // Number of routines to use for hashing
  47. Hashers int
  48. // Our vector clock id
  49. ShortID protocol.ShortID
  50. // Optional progress tick interval which defines how often FolderScanProgress
  51. // events are emitted. Negative number means disabled.
  52. ProgressTickIntervalS int
  53. // Local flags to set on scanned files
  54. LocalFlags protocol.FlagLocal
  55. // Modification time is to be considered unchanged if the difference is lower.
  56. ModTimeWindow time.Duration
  57. // Event logger to which the scan progress events are sent
  58. EventLogger events.Logger
  59. // If ScanOwnership is true, we pick up ownership information on files while scanning.
  60. ScanOwnership bool
  61. // If ScanXattrs is true, we pick up extended attributes on files while scanning.
  62. ScanXattrs bool
  63. // Filter for extended attributes
  64. XattrFilter XattrFilter
  65. }
  66. type CurrentFiler interface {
  67. // CurrentFile returns the file as seen at last scan.
  68. CurrentFile(name string) (protocol.FileInfo, bool)
  69. }
  70. type XattrFilter interface {
  71. Permit(string) bool
  72. GetMaxSingleEntrySize() int
  73. GetMaxTotalSize() int
  74. }
  75. type ScanResult struct {
  76. File protocol.FileInfo
  77. Err error
  78. Path string // to be set in case Err != nil and File == nil
  79. }
  80. func Walk(ctx context.Context, cfg Config) chan ScanResult {
  81. return newWalker(cfg).walk(ctx)
  82. }
  83. func WalkWithoutHashing(ctx context.Context, cfg Config) chan ScanResult {
  84. return newWalker(cfg).walkWithoutHashing(ctx)
  85. }
  86. func newWalker(cfg Config) *walker {
  87. w := &walker{cfg}
  88. if w.CurrentFiler == nil {
  89. w.CurrentFiler = noCurrentFiler{}
  90. }
  91. if w.Filesystem == nil {
  92. panic("no filesystem specified")
  93. }
  94. if w.Matcher == nil {
  95. w.Matcher = ignore.New(w.Filesystem)
  96. }
  97. registerFolderMetrics(w.Folder)
  98. return w
  99. }
  100. var (
  101. errUTF8Invalid = errors.New("item is not in UTF8 encoding")
  102. errUTF8Normalization = errors.New("item is not in the correct UTF8 normalization form")
  103. errUTF8Conflict = errors.New("item has UTF8 encoding conflict with another item")
  104. )
  105. type walker struct {
  106. Config
  107. }
  108. // Walk returns the list of files found in the local folder by scanning the
  109. // file system. Files are blockwise hashed.
  110. func (w *walker) walk(ctx context.Context) chan ScanResult {
  111. l.Debugln(w, "Walk", w.Subs, w.Matcher)
  112. toHashChan := make(chan protocol.FileInfo)
  113. finishedChan := make(chan ScanResult)
  114. // A routine which walks the filesystem tree, and sends files which have
  115. // been modified to the counter routine.
  116. go w.scan(ctx, toHashChan, finishedChan)
  117. // We're not required to emit scan progress events, just kick off hashers,
  118. // and feed inputs directly from the walker.
  119. if w.ProgressTickIntervalS < 0 {
  120. newParallelHasher(ctx, w.Folder, w.Filesystem, w.Hashers, finishedChan, toHashChan, nil, nil)
  121. return finishedChan
  122. }
  123. // Defaults to every 2 seconds.
  124. if w.ProgressTickIntervalS == 0 {
  125. w.ProgressTickIntervalS = 2
  126. }
  127. // We need to emit progress events, hence we create a routine which buffers
  128. // the list of files to be hashed, counts the total number of
  129. // bytes to hash, and once no more files need to be hashed (chan gets closed),
  130. // start a routine which periodically emits FolderScanProgress events,
  131. // until a stop signal is sent by the parallel hasher.
  132. // Parallel hasher is stopped by this routine when we close the channel over
  133. // which it receives the files we ask it to hash.
  134. go func() {
  135. var filesToHash []protocol.FileInfo
  136. var total int64 = 1
  137. for file := range toHashChan {
  138. filesToHash = append(filesToHash, file)
  139. total += file.Size
  140. }
  141. if len(filesToHash) == 0 {
  142. close(finishedChan)
  143. return
  144. }
  145. realToHashChan := make(chan protocol.FileInfo)
  146. done := make(chan struct{})
  147. progress := newByteCounter()
  148. newParallelHasher(ctx, w.Folder, w.Filesystem, w.Hashers, finishedChan, realToHashChan, progress, done)
  149. // A routine which actually emits the FolderScanProgress events
  150. // every w.ProgressTicker ticks, until the hasher routines terminate.
  151. go func() {
  152. defer progress.Close()
  153. emitProgressEvent := func() {
  154. current := progress.Total()
  155. rate := progress.Rate()
  156. l.Debugf("%v: Walk %s %s current progress %d/%d at %.01f MiB/s (%d%%)", w, w.Folder, w.Subs, current, total, rate/1024/1024, current*100/total)
  157. w.EventLogger.Log(events.FolderScanProgress, map[string]interface{}{
  158. "folder": w.Folder,
  159. "current": current,
  160. "total": total,
  161. "rate": rate, // bytes per second
  162. })
  163. }
  164. ticker := time.NewTicker(time.Duration(w.ProgressTickIntervalS) * time.Second)
  165. defer ticker.Stop()
  166. for {
  167. select {
  168. case <-done:
  169. emitProgressEvent()
  170. l.Debugln(w, "Walk progress done", w.Folder, w.Subs, w.Matcher)
  171. return
  172. case <-ticker.C:
  173. emitProgressEvent()
  174. case <-ctx.Done():
  175. return
  176. }
  177. }
  178. }()
  179. loop:
  180. for _, file := range filesToHash {
  181. l.Debugln(w, "real to hash:", file.Name)
  182. select {
  183. case realToHashChan <- file:
  184. case <-ctx.Done():
  185. break loop
  186. }
  187. }
  188. close(realToHashChan)
  189. }()
  190. return finishedChan
  191. }
  192. func (w *walker) walkWithoutHashing(ctx context.Context) chan ScanResult {
  193. l.Debugln(w, "Walk without hashing", w.Subs, w.Matcher)
  194. toHashChan := make(chan protocol.FileInfo)
  195. finishedChan := make(chan ScanResult)
  196. // A routine which walks the filesystem tree, and sends files which have
  197. // been modified to the counter routine.
  198. go w.scan(ctx, toHashChan, finishedChan)
  199. go func() {
  200. for file := range toHashChan {
  201. finishedChan <- ScanResult{File: file}
  202. }
  203. close(finishedChan)
  204. }()
  205. return finishedChan
  206. }
  207. const walkFailureEventDesc = "Unexpected error while walking the filesystem during scan"
  208. func (w *walker) scan(ctx context.Context, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) {
  209. hashFiles := w.walkAndHashFiles(ctx, toHashChan, finishedChan)
  210. if len(w.Subs) == 0 {
  211. if err := w.Filesystem.Walk(".", hashFiles); isWarnableError(err) {
  212. w.EventLogger.Log(events.Failure, walkFailureEventDesc)
  213. slog.ErrorContext(ctx, "Aborted scan due to an unexpected error", slogutil.Error(err))
  214. }
  215. } else {
  216. for _, sub := range w.Subs {
  217. if err := osutil.TraversesSymlink(w.Filesystem, filepath.Dir(sub)); err != nil {
  218. l.Debugf("%v: Skip walking %v as it is below a symlink", w, sub)
  219. continue
  220. }
  221. if err := w.Filesystem.Walk(sub, hashFiles); isWarnableError(err) {
  222. w.EventLogger.Log(events.Failure, walkFailureEventDesc)
  223. slog.ErrorContext(ctx, "Aborted scan due to an unexpected error", slogutil.FilePath(sub), slogutil.Error(err))
  224. }
  225. }
  226. }
  227. close(toHashChan)
  228. }
  229. // isWarnableError returns true if err is a kind of error we should warn
  230. // about receiving from the folder walk.
  231. func isWarnableError(err error) bool {
  232. return err != nil &&
  233. !errors.Is(err, fs.SkipDir) && // intentional skip
  234. !errors.Is(err, context.Canceled) // folder restarting
  235. }
  236. func (w *walker) walkAndHashFiles(ctx context.Context, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) fs.WalkFunc {
  237. now := time.Now()
  238. ignoredParent := ""
  239. return func(path string, info fs.FileInfo, err error) error {
  240. select {
  241. case <-ctx.Done():
  242. return ctx.Err()
  243. default:
  244. }
  245. metricScannedItems.WithLabelValues(w.Folder).Inc()
  246. // Return value used when we are returning early and don't want to
  247. // process the item. For directories, this means do-not-descend.
  248. var skip error // nil
  249. // info nil when error is not nil
  250. if info != nil && info.IsDir() {
  251. skip = fs.SkipDir
  252. }
  253. if !utf8.ValidString(path) {
  254. handleError(ctx, "scan", path, errUTF8Invalid, finishedChan)
  255. return skip
  256. }
  257. if fs.IsTemporary(path) {
  258. l.Debugln(w, "temporary:", path, "err:", err)
  259. if err == nil && info.IsRegular() && info.ModTime().Add(w.TempLifetime).Before(now) {
  260. w.Filesystem.Remove(path)
  261. l.Debugln(w, "removing temporary:", path, info.ModTime())
  262. }
  263. return nil
  264. }
  265. if fs.IsInternal(path) {
  266. l.Debugln(w, "ignored (internal):", path)
  267. return skip
  268. }
  269. // Just in case the filesystem doesn't produce the normalization the OS
  270. // uses, and we use internally.
  271. nonNormPath := path
  272. path = normalizePath(path)
  273. if m := w.Matcher.Match(path); m.IsIgnored() {
  274. l.Debugln(w, "ignored (patterns):", path)
  275. // Only descend if matcher says so and the current file is not a symlink.
  276. if err != nil || m.CanSkipDir() || info.IsSymlink() {
  277. return skip
  278. }
  279. // If the parent wasn't ignored already, set this path as the "highest" ignored parent
  280. if info.IsDir() && (ignoredParent == "" || !fs.IsParent(path, ignoredParent)) {
  281. ignoredParent = path
  282. }
  283. return nil
  284. }
  285. if err != nil {
  286. // No need reporting errors for files that don't exist (e.g. scan
  287. // due to filesystem watcher)
  288. if !fs.IsNotExist(err) {
  289. handleError(ctx, "scan", path, err, finishedChan)
  290. }
  291. return skip
  292. }
  293. if path == "." {
  294. return nil
  295. }
  296. if path != nonNormPath {
  297. if !w.AutoNormalize {
  298. // We're not authorized to do anything about it, so complain and skip.
  299. handleError(ctx, "normalizing path", nonNormPath, errUTF8Normalization, finishedChan)
  300. return skip
  301. }
  302. path, err = w.applyNormalization(nonNormPath, path, info)
  303. if err != nil {
  304. handleError(ctx, "normalizing path", nonNormPath, err, finishedChan)
  305. return skip
  306. }
  307. }
  308. if ignoredParent == "" {
  309. // parent isn't ignored, nothing special
  310. if err := w.handleItem(ctx, path, info, toHashChan, finishedChan); err != nil {
  311. handleError(ctx, "scan", path, err, finishedChan)
  312. return skip
  313. }
  314. return nil
  315. }
  316. // Part of current path below the ignored (potential) parent
  317. rel := strings.TrimPrefix(path, ignoredParent+string(fs.PathSeparator))
  318. // ignored path isn't actually a parent of the current path
  319. if rel == path {
  320. ignoredParent = ""
  321. if err := w.handleItem(ctx, path, info, toHashChan, finishedChan); err != nil {
  322. handleError(ctx, "scan", path, err, finishedChan)
  323. return skip
  324. }
  325. return nil
  326. }
  327. // The previously ignored parent directories of the current, not
  328. // ignored path need to be handled as well.
  329. // Prepend an empty string to handle ignoredParent without anything
  330. // appended in the first iteration.
  331. for _, name := range append([]string{""}, fs.PathComponents(rel)...) {
  332. ignoredParent = filepath.Join(ignoredParent, name)
  333. info, err = w.Filesystem.Lstat(ignoredParent)
  334. // An error here would be weird as we've already gotten to this point, but act on it nonetheless
  335. if err != nil {
  336. handleError(ctx, "scan", ignoredParent, err, finishedChan)
  337. return skip
  338. }
  339. if err = w.handleItem(ctx, ignoredParent, info, toHashChan, finishedChan); err != nil {
  340. handleError(ctx, "scan", path, err, finishedChan)
  341. return skip
  342. }
  343. }
  344. ignoredParent = ""
  345. return nil
  346. }
  347. }
  348. // Returning an error does not indicate that the walk should be aborted - it
  349. // will simply report the error for that path to the user (same for walk...
  350. // functions called from here).
  351. func (w *walker) handleItem(ctx context.Context, path string, info fs.FileInfo, toHashChan chan<- protocol.FileInfo, finishedChan chan<- ScanResult) error {
  352. switch {
  353. case info.IsSymlink():
  354. if err := w.walkSymlink(ctx, path, info, finishedChan); err != nil {
  355. return err
  356. }
  357. if info.IsDir() {
  358. // under no circumstances shall we descend into a symlink
  359. return fs.SkipDir
  360. }
  361. return nil
  362. case info.IsDir():
  363. return w.walkDir(ctx, path, info, finishedChan)
  364. case info.IsRegular():
  365. return w.walkRegular(ctx, path, info, toHashChan)
  366. default:
  367. // A special file, socket, fifo, etc. -- do nothing, just skip and continue scanning.
  368. l.Debugf("Skipping non-regular file %s (%s)", path, info.Mode())
  369. return nil
  370. }
  371. }
  372. func (w *walker) walkRegular(ctx context.Context, relPath string, info fs.FileInfo, toHashChan chan<- protocol.FileInfo) error {
  373. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  374. blockSize := protocol.BlockSize(info.Size())
  375. if hasCurFile {
  376. // Check if we should retain current block size.
  377. curBlockSize := curFile.BlockSize()
  378. if blockSize > curBlockSize && blockSize/curBlockSize <= 2 {
  379. // New block size is larger, but not more than twice larger.
  380. // Retain.
  381. blockSize = curBlockSize
  382. } else if curBlockSize > blockSize && curBlockSize/blockSize <= 2 {
  383. // Old block size is larger, but not more than twice larger.
  384. // Retain.
  385. blockSize = curBlockSize
  386. }
  387. }
  388. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  389. if err != nil {
  390. return err
  391. }
  392. f = w.updateFileInfo(f, curFile)
  393. f.NoPermissions = w.IgnorePerms
  394. f.RawBlockSize = int32(blockSize)
  395. l.Debugln(w, "checking:", f)
  396. if hasCurFile {
  397. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  398. ModTimeWindow: w.ModTimeWindow,
  399. IgnorePerms: w.IgnorePerms,
  400. IgnoreBlocks: true,
  401. IgnoreFlags: w.LocalFlags,
  402. IgnoreOwnership: !w.ScanOwnership,
  403. IgnoreXattrs: !w.ScanXattrs,
  404. }) {
  405. l.Debugln(w, "unchanged:", curFile)
  406. return nil
  407. }
  408. if curFile.ShouldConflict() && !f.ShouldConflict() {
  409. // The old file was invalid for whatever reason and probably not
  410. // up to date with what was out there in the cluster. Drop all
  411. // others from the version vector to indicate that we haven't
  412. // taken their version into account, and possibly cause a
  413. // conflict. However, only do this if the new file is not also
  414. // invalid. This would indicate that the new file is not part
  415. // of the cluster, but e.g. a local change.
  416. f.Version = f.Version.DropOthers(w.ShortID)
  417. }
  418. l.Debugln(w, "rescan:", curFile)
  419. }
  420. l.Debugln(w, "to hash:", relPath, f)
  421. select {
  422. case toHashChan <- f:
  423. case <-ctx.Done():
  424. return ctx.Err()
  425. }
  426. return nil
  427. }
  428. func (w *walker) walkDir(ctx context.Context, relPath string, info fs.FileInfo, finishedChan chan<- ScanResult) error {
  429. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  430. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  431. if err != nil {
  432. return err
  433. }
  434. f = w.updateFileInfo(f, curFile)
  435. f.NoPermissions = w.IgnorePerms
  436. l.Debugln(w, "checking:", f)
  437. if hasCurFile {
  438. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  439. ModTimeWindow: w.ModTimeWindow,
  440. IgnorePerms: w.IgnorePerms,
  441. IgnoreBlocks: true,
  442. IgnoreFlags: w.LocalFlags,
  443. IgnoreOwnership: !w.ScanOwnership,
  444. IgnoreXattrs: !w.ScanXattrs,
  445. }) {
  446. l.Debugln(w, "unchanged:", curFile)
  447. return nil
  448. }
  449. if curFile.ShouldConflict() && !f.ShouldConflict() {
  450. // The old file was invalid for whatever reason and probably not
  451. // up to date with what was out there in the cluster. Drop all
  452. // others from the version vector to indicate that we haven't
  453. // taken their version into account, and possibly cause a
  454. // conflict. However, only do this if the new file is not also
  455. // invalid. This would indicate that the new file is not part
  456. // of the cluster, but e.g. a local change.
  457. f.Version = f.Version.DropOthers(w.ShortID)
  458. }
  459. l.Debugln(w, "rescan:", curFile)
  460. }
  461. l.Debugln(w, "dir:", relPath, f)
  462. select {
  463. case finishedChan <- ScanResult{File: f}:
  464. case <-ctx.Done():
  465. return ctx.Err()
  466. }
  467. return nil
  468. }
  469. func (w *walker) walkSymlink(ctx context.Context, relPath string, info fs.FileInfo, finishedChan chan<- ScanResult) error {
  470. // Symlinks are not supported on Windows. We ignore instead of returning
  471. // an error.
  472. if build.IsWindows {
  473. return nil
  474. }
  475. f, err := CreateFileInfo(info, relPath, w.Filesystem, w.ScanOwnership, w.ScanXattrs, w.XattrFilter)
  476. if err != nil {
  477. return err
  478. }
  479. curFile, hasCurFile := w.CurrentFiler.CurrentFile(relPath)
  480. f = w.updateFileInfo(f, curFile)
  481. l.Debugln(w, "checking:", f)
  482. if hasCurFile {
  483. if curFile.IsEquivalentOptional(f, protocol.FileInfoComparison{
  484. ModTimeWindow: w.ModTimeWindow,
  485. IgnorePerms: w.IgnorePerms,
  486. IgnoreBlocks: true,
  487. IgnoreFlags: w.LocalFlags,
  488. IgnoreOwnership: !w.ScanOwnership,
  489. IgnoreXattrs: !w.ScanXattrs,
  490. }) {
  491. l.Debugln(w, "unchanged:", curFile, info.ModTime().Unix(), info.Mode()&fs.ModePerm)
  492. return nil
  493. }
  494. if curFile.ShouldConflict() && !f.ShouldConflict() {
  495. // The old file was invalid for whatever reason and probably not
  496. // up to date with what was out there in the cluster. Drop all
  497. // others from the version vector to indicate that we haven't
  498. // taken their version into account, and possibly cause a
  499. // conflict. However, only do this if the new file is not also
  500. // invalid. This would indicate that the new file is not part
  501. // of the cluster, but e.g. a local change.
  502. f.Version = f.Version.DropOthers(w.ShortID)
  503. }
  504. l.Debugln(w, "rescan:", curFile)
  505. }
  506. l.Debugln(w, "symlink:", relPath, f)
  507. select {
  508. case finishedChan <- ScanResult{File: f}:
  509. case <-ctx.Done():
  510. return ctx.Err()
  511. }
  512. return nil
  513. }
  514. func normalizePath(path string) string {
  515. if build.IsDarwin || build.IsIOS {
  516. // Mac OS X file names should always be NFD normalized.
  517. return norm.NFD.String(path)
  518. }
  519. // Every other OS in the known universe uses NFC or just plain
  520. // doesn't bother to define an encoding. In our case *we* do care,
  521. // so we enforce NFC regardless.
  522. return norm.NFC.String(path)
  523. }
  524. // applyNormalization fixes the normalization of the file on disk, i.e. ensures
  525. // the file at path ends up named normPath. It shouldn't but may happen that the
  526. // file ends up with a different name, in which case that one should be scanned.
  527. func (w *walker) applyNormalization(path, normPath string, info fs.FileInfo) (string, error) {
  528. // We will attempt to normalize it.
  529. normInfo, err := w.Filesystem.Lstat(normPath)
  530. if fs.IsNotExist(err) {
  531. // Nothing exists with the normalized filename. Good.
  532. if err = w.Filesystem.Rename(path, normPath); err != nil {
  533. return "", err
  534. }
  535. slog.Info("Normalized UTF8 encoding of file name", slogutil.FilePath(path))
  536. return normPath, nil
  537. }
  538. if w.Filesystem.SameFile(info, normInfo) {
  539. // With some filesystems (ZFS), if there is an un-normalized path and you ask whether the normalized
  540. // version exists, it responds with true. Therefore we need to check fs.SameFile as well.
  541. // In this case, a call to Rename won't do anything, so we have to rename via a temp file.
  542. // We don't want to use the standard syncthing prefix here, as that will result in the file being ignored
  543. // and eventually deleted by Syncthing if the rename back fails.
  544. tempPath := fs.TempNameWithPrefix(normPath, "")
  545. if err = w.Filesystem.Rename(path, tempPath); err != nil {
  546. return "", err
  547. }
  548. if err = w.Filesystem.Rename(tempPath, normPath); err != nil {
  549. // I don't ever expect this to happen, but if it does, we should probably tell our caller that the normalized
  550. // path is the temp path: that way at least the user's data still gets synced.
  551. slog.Error("Failed to rename while normalizating UTF8 encoding; please rename temp file manually", slog.String("from", tempPath), slog.String("to", normPath), slogutil.Error(err))
  552. return tempPath, nil
  553. }
  554. return normPath, nil
  555. }
  556. // There is something already in the way at the normalized
  557. // file name.
  558. return "", errUTF8Conflict
  559. }
  560. // updateFileInfo updates walker specific members of protocol.FileInfo that
  561. // do not depend on type, and things that should be preserved from the
  562. // previous version of the FileInfo.
  563. func (w *walker) updateFileInfo(dst, src protocol.FileInfo) protocol.FileInfo {
  564. if dst.Type == protocol.FileInfoTypeFile && build.IsWindows {
  565. // If we have an existing index entry, copy the executable bits
  566. // from there.
  567. dst.Permissions |= (src.Permissions & 0o111)
  568. }
  569. dst.Version = src.Version.Update(w.ShortID)
  570. dst.ModifiedBy = w.ShortID
  571. dst.LocalFlags = w.LocalFlags
  572. dst.PreviousBlocksHash = src.BlocksHash
  573. // Copy OS data from src to dst, unless it was already set on dst.
  574. dst.Platform.MergeWith(&src.Platform)
  575. return dst
  576. }
  577. func handleError(ctx context.Context, context, path string, err error, finishedChan chan<- ScanResult) {
  578. l.Debugf("handle error on '%v': %v: %v", path, context, err)
  579. select {
  580. case finishedChan <- ScanResult{
  581. Err: fmt.Errorf("%s: %w", context, err),
  582. Path: path,
  583. }:
  584. case <-ctx.Done():
  585. }
  586. }
  587. func (w *walker) String() string {
  588. return fmt.Sprintf("walker/%s@%p", w.Folder, w)
  589. }
  590. // A byteCounter gets bytes added to it via Update() and then provides the
  591. // Total() and one minute moving average Rate() in bytes per second.
  592. type byteCounter struct {
  593. total atomic.Int64
  594. metrics.EWMA
  595. stop chan struct{}
  596. }
  597. func newByteCounter() *byteCounter {
  598. c := &byteCounter{
  599. EWMA: metrics.NewEWMA1(), // a one minute exponentially weighted moving average
  600. stop: make(chan struct{}),
  601. }
  602. go c.ticker()
  603. return c
  604. }
  605. func (c *byteCounter) ticker() {
  606. // The metrics.EWMA expects clock ticks every five seconds in order to
  607. // decay the average properly.
  608. t := time.NewTicker(5 * time.Second)
  609. for {
  610. select {
  611. case <-t.C:
  612. c.Tick()
  613. case <-c.stop:
  614. t.Stop()
  615. return
  616. }
  617. }
  618. }
  619. func (c *byteCounter) Update(bytes int64) {
  620. c.total.Add(bytes)
  621. c.EWMA.Update(bytes)
  622. }
  623. func (c *byteCounter) Total() int64 { return c.total.Load() }
  624. func (c *byteCounter) Close() {
  625. close(c.stop)
  626. }
  627. // A no-op CurrentFiler
  628. type noCurrentFiler struct{}
  629. func (noCurrentFiler) CurrentFile(_ string) (protocol.FileInfo, bool) {
  630. return protocol.FileInfo{}, false
  631. }
  632. func CreateFileInfo(fi fs.FileInfo, name string, filesystem fs.Filesystem, scanOwnership bool, scanXattrs bool, xattrFilter XattrFilter) (protocol.FileInfo, error) {
  633. f := protocol.FileInfo{Name: name}
  634. if scanOwnership || scanXattrs {
  635. if plat, err := filesystem.PlatformData(name, scanOwnership, scanXattrs, xattrFilter); err == nil {
  636. f.Platform = plat
  637. } else {
  638. return protocol.FileInfo{}, fmt.Errorf("reading platform data: %w", err)
  639. }
  640. }
  641. if ct := fi.InodeChangeTime(); !ct.IsZero() {
  642. f.InodeChangeNs = ct.UnixNano()
  643. } else {
  644. f.InodeChangeNs = 0
  645. }
  646. if fi.IsSymlink() {
  647. f.Type = protocol.FileInfoTypeSymlink
  648. target, err := filesystem.ReadSymlink(name)
  649. if err != nil {
  650. return protocol.FileInfo{}, err
  651. }
  652. f.SymlinkTarget = []byte(target)
  653. f.NoPermissions = true // Symlinks don't have permissions of their own
  654. return f, nil
  655. }
  656. f.Permissions = uint32(fi.Mode() & fs.ModePerm)
  657. f.ModifiedS = fi.ModTime().Unix()
  658. f.ModifiedNs = int32(fi.ModTime().Nanosecond())
  659. if fi.IsDir() {
  660. f.Type = protocol.FileInfoTypeDirectory
  661. return f, nil
  662. }
  663. f.Size = fi.Size()
  664. f.Type = protocol.FileInfoTypeFile
  665. return f, nil
  666. }