Merge pull request #953 from syncthing/symlink

Symlink support

internal/files/blockmap.go

@@ -29,6 +29,7 @@ import (
 	"sync"
 
 	"github.com/syncthing/syncthing/internal/config"
+	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
 
 	"github.com/syndtr/goleveldb/leveldb"
@@ -171,7 +172,7 @@ func (f *BlockFinder) Iterate(hash []byte, iterFn func(string, string, uint32) b
 		for iter.Next() && iter.Error() == nil {
 			folder, file := fromBlockKey(iter.Key())
 			index := binary.BigEndian.Uint32(iter.Value())
-			if iterFn(folder, nativeFilename(file), index) {
+			if iterFn(folder, osutil.NativeFilename(file), index) {
 				return true
 			}
 		}

internal/files/set.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 
 	"github.com/syncthing/syncthing/internal/lamport"
+	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -174,19 +175,19 @@ func (s *Set) WithGlobalTruncated(fn fileIterator) {
 }
 
 func (s *Set) Get(device protocol.DeviceID, file string) protocol.FileInfo {
-	f := ldbGet(s.db, []byte(s.folder), device[:], []byte(normalizedFilename(file)))
-	f.Name = nativeFilename(f.Name)
+	f := ldbGet(s.db, []byte(s.folder), device[:], []byte(osutil.NormalizedFilename(file)))
+	f.Name = osutil.NativeFilename(f.Name)
 	return f
 }
 
 func (s *Set) GetGlobal(file string) protocol.FileInfo {
-	f := ldbGetGlobal(s.db, []byte(s.folder), []byte(normalizedFilename(file)))
-	f.Name = nativeFilename(f.Name)
+	f := ldbGetGlobal(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
+	f.Name = osutil.NativeFilename(f.Name)
 	return f
 }
 
 func (s *Set) Availability(file string) []protocol.DeviceID {
-	return ldbAvailability(s.db, []byte(s.folder), []byte(normalizedFilename(file)))
+	return ldbAvailability(s.db, []byte(s.folder), []byte(osutil.NormalizedFilename(file)))
 }
 
 func (s *Set) LocalVersion(device protocol.DeviceID) uint64 {
@@ -213,7 +214,7 @@ func DropFolder(db *leveldb.DB, folder string) {
 
 func normalizeFilenames(fs []protocol.FileInfo) {
 	for i := range fs {
-		fs[i].Name = normalizedFilename(fs[i].Name)
+		fs[i].Name = osutil.NormalizedFilename(fs[i].Name)
 	}
 }
 
@@ -221,10 +222,10 @@ func nativeFileIterator(fn fileIterator) fileIterator {
 	return func(fi protocol.FileIntf) bool {
 		switch f := fi.(type) {
 		case protocol.FileInfo:
-			f.Name = nativeFilename(f.Name)
+			f.Name = osutil.NativeFilename(f.Name)
 			return fn(f)
 		case protocol.FileInfoTruncated:
-			f.Name = nativeFilename(f.Name)
+			f.Name = osutil.NativeFilename(f.Name)
 			return fn(f)
 		default:
 			panic("unknown interface type")

internal/model/model.go

@@ -39,6 +39,7 @@ import (
 	"github.com/syncthing/syncthing/internal/protocol"
 	"github.com/syncthing/syncthing/internal/scanner"
 	"github.com/syncthing/syncthing/internal/stats"
+	"github.com/syncthing/syncthing/internal/symlinks"
 	"github.com/syncthing/syncthing/internal/versioner"
 	"github.com/syndtr/goleveldb/leveldb"
 )
@@ -114,6 +115,8 @@ type Model struct {
 var (
 	ErrNoSuchFile = errors.New("no such file")
 	ErrInvalid    = errors.New("file is invalid")
+
+	SymlinkWarning = sync.Once{}
 )
 
 // NewModel creates and starts a new model. The model starts in read-only mode,
@@ -440,9 +443,9 @@ func (m *Model) Index(deviceID protocol.DeviceID, folder string, fs []protocol.F
 
 	for i := 0; i < len(fs); {
 		lamport.Default.Tick(fs[i].Version)
-		if ignores != nil && ignores.Match(fs[i].Name) {
+		if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
 			if debug {
-				l.Debugln("dropping update for ignored", fs[i])
+				l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
 			}
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
@@ -484,9 +487,9 @@ func (m *Model) IndexUpdate(deviceID protocol.DeviceID, folder string, fs []prot
 
 	for i := 0; i < len(fs); {
 		lamport.Default.Tick(fs[i].Version)
-		if ignores != nil && ignores.Match(fs[i].Name) {
+		if (ignores != nil && ignores.Match(fs[i].Name)) || symlinkInvalid(fs[i].IsSymlink()) {
 			if debug {
-				l.Debugln("dropping update for ignored", fs[i])
+				l.Debugln("dropping update for ignored/unsupported symlink", fs[i])
 			}
 			fs[i] = fs[len(fs)-1]
 			fs = fs[:len(fs)-1]
@@ -655,7 +658,7 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 	}
 
 	lf := r.Get(protocol.LocalDeviceID, name)
-	if protocol.IsInvalid(lf.Flags) || protocol.IsDeleted(lf.Flags) {
+	if lf.IsInvalid() || lf.IsDeleted() {
 		if debug {
 			l.Debugf("%v REQ(in): %s: %q / %q o=%d s=%d; invalid: %v", m, deviceID, folder, name, offset, size, lf)
 		}
@@ -675,14 +678,26 @@ func (m *Model) Request(deviceID protocol.DeviceID, folder, name string, offset
 	m.fmut.RLock()
 	fn := filepath.Join(m.folderCfgs[folder].Path, name)
 	m.fmut.RUnlock()
-	fd, err := os.Open(fn) // XXX: Inefficient, should cache fd?
-	if err != nil {
-		return nil, err
+
+	var reader io.ReaderAt
+	var err error
+	if lf.IsSymlink() {
+		target, _, err := symlinks.Read(fn)
+		if err != nil {
+			return nil, err
+		}
+		reader = strings.NewReader(target)
+	} else {
+		reader, err = os.Open(fn) // XXX: Inefficient, should cache fd?
+		if err != nil {
+			return nil, err
+		}
+
+		defer reader.(*os.File).Close()
 	}
-	defer fd.Close()
 
 	buf := make([]byte, size)
-	_, err = fd.ReadAt(buf, offset)
+	_, err = reader.ReadAt(buf, offset)
 	if err != nil {
 		return nil, err
 	}
@@ -892,9 +907,9 @@ func sendIndexTo(initial bool, minLocalVer uint64, conn protocol.Connection, fol
 			maxLocalVer = f.LocalVersion
 		}
 
-		if ignores != nil && ignores.Match(f.Name) {
+		if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
 			if debug {
-				l.Debugln("not sending update for ignored", f)
+				l.Debugln("not sending update for ignored/unsupported symlink", f)
 			}
 			return true
 		}
@@ -1085,7 +1100,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 		}
 
 		seenPrefix = true
-		if !protocol.IsDeleted(f.Flags) {
+		if !f.IsDeleted() {
 			if f.IsInvalid() {
 				return true
 			}
@@ -1095,8 +1110,8 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 				batch = batch[:0]
 			}
 
-			if ignores != nil && ignores.Match(f.Name) {
-				// File has been ignored. Set invalid bit.
+			if (ignores != nil && ignores.Match(f.Name)) || symlinkInvalid(f.IsSymlink()) {
+				// File has been ignored or an unsupported symlink. Set invalid bit.
 				l.Debugln("setting invalid bit on ignored", f)
 				nf := protocol.FileInfo{
 					Name:     f.Name,
@@ -1112,7 +1127,7 @@ func (m *Model) ScanFolderSub(folder, sub string) error {
 					"size":     f.Size(),
 				})
 				batch = append(batch, nf)
-			} else if _, err := os.Stat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
+			} else if _, err := os.Lstat(filepath.Join(dir, f.Name)); err != nil && os.IsNotExist(err) {
 				// File has been deleted
 				nf := protocol.FileInfo{
 					Name:     f.Name,
@@ -1326,3 +1341,13 @@ func (m *Model) leveldbPanicWorkaround() {
 		}
 	}
 }
+
+func symlinkInvalid(isLink bool) bool {
+	if !symlinks.Supported && isLink {
+		SymlinkWarning.Do(func() {
+			l.Warnln("Symlinks are unsupported as they require Administrator priviledges. This might cause your folder to appear out of sync.")
+		})
+		return true
+	}
+	return false
+}

internal/model/puller.go

@@ -20,6 +20,7 @@ import (
 	"crypto/sha256"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -32,6 +33,7 @@ import (
 	"github.com/syncthing/syncthing/internal/osutil"
 	"github.com/syncthing/syncthing/internal/protocol"
 	"github.com/syncthing/syncthing/internal/scanner"
+	"github.com/syncthing/syncthing/internal/symlinks"
 	"github.com/syncthing/syncthing/internal/versioner"
 )
 
@@ -313,15 +315,16 @@ func (p *Puller) pullerIteration(ncopiers, npullers, nfinishers int, checksum bo
 		}
 
 		switch {
-		case protocol.IsDeleted(file.Flags):
-			// A deleted file or directory
+		case file.IsDeleted():
+			// A deleted file, directory or symlink
 			deletions = append(deletions, file)
-		case protocol.IsDirectory(file.Flags):
+		case file.IsDirectory() && !file.IsSymlink():
 			// A new or changed directory
 			p.handleDir(file)
 		default:
-			// A new or changed file. This is the only case where we do stuff
-			// in the background; the other three are done synchronously.
+			// A new or changed file or symlink. This is the only case where we
+			// do stuff in the background; the other three are done
+			// synchronously.
 			p.handleFile(file, copyChan, finisherChan)
 		}
 
@@ -459,24 +462,21 @@ func (p *Puller) deleteFile(file protocol.FileInfo) {
 func (p *Puller) handleFile(file protocol.FileInfo, copyChan chan<- copyBlocksState, finisherChan chan<- *sharedPullerState) {
 	curFile := p.model.CurrentFolderFile(p.folder, file.Name)
 
-	if len(curFile.Blocks) == len(file.Blocks) {
-		for i := range file.Blocks {
-			if !bytes.Equal(curFile.Blocks[i].Hash, file.Blocks[i].Hash) {
-				goto FilesAreDifferent
-			}
-		}
+	if len(curFile.Blocks) == len(file.Blocks) && scanner.BlocksEqual(curFile.Blocks, file.Blocks) {
 		// We are supposed to copy the entire file, and then fetch nothing. We
 		// are only updating metadata, so we don't actually *need* to make the
 		// copy.
 		if debug {
 			l.Debugln(p, "taking shortcut on", file.Name)
 		}
-		p.shortcutFile(file)
+		if file.IsSymlink() {
+			p.shortcutSymlink(curFile, file)
+		} else {
+			p.shortcutFile(file)
+		}
 		return
 	}
 
-FilesAreDifferent:
-
 	scanner.PopulateOffsets(file.Blocks)
 
 	// Figure out the absolute filenames we need once and for all
@@ -571,6 +571,17 @@ func (p *Puller) shortcutFile(file protocol.FileInfo) {
 	p.model.updateLocal(p.folder, file)
 }
 
+// shortcutSymlink changes the symlinks type if necessery.
+func (p *Puller) shortcutSymlink(curFile, file protocol.FileInfo) {
+	err := symlinks.ChangeType(filepath.Join(p.dir, file.Name), file.Flags)
+	if err != nil {
+		l.Infof("Puller (folder %q, file %q): symlink shortcut: %v", p.folder, file.Name, err)
+		return
+	}
+
+	p.model.updateLocal(p.folder, file)
+}
+
 // copierRoutine reads copierStates until the in channel closes and performs
 // the relevant copies when possible, or passes it to the puller routine.
 func (p *Puller) copierRoutine(in <-chan copyBlocksState, pullChan chan<- pullBlockState, out chan<- *sharedPullerState, checksum bool) {
@@ -791,6 +802,25 @@ func (p *Puller) finisherRoutine(in <-chan *sharedPullerState) {
 				continue
 			}
 
+			// If it's a symlink, the target of the symlink is inside the file.
+			if state.file.IsSymlink() {
+				content, err := ioutil.ReadFile(state.realName)
+				if err != nil {
+					l.Warnln("puller: final: reading symlink:", err)
+					continue
+				}
+
+				// Remove the file, and replace it with a symlink.
+				err = osutil.InWritableDir(func(path string) error {
+					os.Remove(path)
+					return symlinks.Create(path, string(content), state.file.Flags)
+				}, state.realName)
+				if err != nil {
+					l.Warnln("puller: final: creating symlink:", err)
+					continue
+				}
+			}
+
 			// Record the updated file in the index
 			p.model.updateLocal(p.folder, state.file)
 		}

internal/files/filenames_darwin.go → internal/osutil/filenames_darwin.go

@@ -13,14 +13,14 @@
 // You should have received a copy of the GNU General Public License along
 // with this program. If not, see <http://www.gnu.org/licenses/>.
 
-package files
+package osutil
 
 import "code.google.com/p/go.text/unicode/norm"
 
-func normalizedFilename(s string) string {
+func NormalizedFilename(s string) string {
 	return norm.NFC.String(s)
 }
 
-func nativeFilename(s string) string {
+func NativeFilename(s string) string {
 	return norm.NFD.String(s)
 }

internal/files/filenames_unix.go → internal/osutil/filenames_unix.go

@@ -15,14 +15,14 @@
 
 // +build !windows,!darwin
 
-package files
+package osutil
 
 import "code.google.com/p/go.text/unicode/norm"
 
-func normalizedFilename(s string) string {
+func NormalizedFilename(s string) string {
 	return norm.NFC.String(s)
 }
 
-func nativeFilename(s string) string {
+func NativeFilename(s string) string {
 	return s
 }

internal/files/filenames_windows.go → internal/osutil/filenames_windows.go

@@ -13,7 +13,7 @@
 // You should have received a copy of the GNU General Public License along
 // with this program. If not, see <http://www.gnu.org/licenses/>.
 
-package files
+package osutil
 
 import (
 	"path/filepath"
@@ -21,10 +21,10 @@ import (
 	"code.google.com/p/go.text/unicode/norm"
 )
 
-func normalizedFilename(s string) string {
+func NormalizedFilename(s string) string {
 	return norm.NFC.String(filepath.ToSlash(s))
 }
 
-func nativeFilename(s string) string {
+func NativeFilename(s string) string {
 	return filepath.FromSlash(s)
 }

internal/protocol/message.go

@@ -37,7 +37,7 @@ func (f FileInfo) String() string {
 }
 
 func (f FileInfo) Size() (bytes int64) {
-	if IsDeleted(f.Flags) || IsDirectory(f.Flags) {
+	if f.IsDeleted() || f.IsDirectory() {
 		return 128
 	}
 	for _, b := range f.Blocks {
@@ -47,15 +47,23 @@ func (f FileInfo) Size() (bytes int64) {
 }
 
 func (f FileInfo) IsDeleted() bool {
-	return IsDeleted(f.Flags)
+	return f.Flags&FlagDeleted != 0
 }
 
 func (f FileInfo) IsInvalid() bool {
-	return IsInvalid(f.Flags)
+	return f.Flags&FlagInvalid != 0
 }
 
 func (f FileInfo) IsDirectory() bool {
-	return IsDirectory(f.Flags)
+	return f.Flags&FlagDirectory != 0
+}
+
+func (f FileInfo) IsSymlink() bool {
+	return f.Flags&FlagSymlink != 0
+}
+
+func (f FileInfo) HasPermissionBits() bool {
+	return f.Flags&FlagNoPermBits == 0
 }
 
 // Used for unmarshalling a FileInfo structure but skipping the actual block list
@@ -75,7 +83,7 @@ func (f FileInfoTruncated) String() string {
 
 // Returns a statistical guess on the size, not the exact figure
 func (f FileInfoTruncated) Size() int64 {
-	if IsDeleted(f.Flags) || IsDirectory(f.Flags) {
+	if f.IsDeleted() || f.IsDirectory() {
 		return 128
 	}
 	if f.NumBlocks < 2 {
@@ -86,17 +94,32 @@ func (f FileInfoTruncated) Size() int64 {
 }
 
 func (f FileInfoTruncated) IsDeleted() bool {
-	return IsDeleted(f.Flags)
+	return f.Flags&FlagDeleted != 0
 }
 
 func (f FileInfoTruncated) IsInvalid() bool {
-	return IsInvalid(f.Flags)
+	return f.Flags&FlagInvalid != 0
+}
+
+func (f FileInfoTruncated) IsDirectory() bool {
+	return f.Flags&FlagDirectory != 0
+}
+
+func (f FileInfoTruncated) IsSymlink() bool {
+	return f.Flags&FlagSymlink != 0
+}
+
+func (f FileInfoTruncated) HasPermissionBits() bool {
+	return f.Flags&FlagNoPermBits == 0
 }
 
 type FileIntf interface {
 	Size() int64
 	IsDeleted() bool
 	IsInvalid() bool
+	IsDirectory() bool
+	IsSymlink() bool
+	HasPermissionBits() bool
 }
 
 type BlockInfo struct {

internal/protocol/protocol.go

@@ -49,10 +49,14 @@ const (
 )
 
 const (
-	FlagDeleted    uint32 = 1 << 12
-	FlagInvalid           = 1 << 13
-	FlagDirectory         = 1 << 14
-	FlagNoPermBits        = 1 << 15
+	FlagDeleted              uint32 = 1 << 12
+	FlagInvalid                     = 1 << 13
+	FlagDirectory                   = 1 << 14
+	FlagNoPermBits                  = 1 << 15
+	FlagSymlink                     = 1 << 16
+	FlagSymlinkMissingTarget        = 1 << 17
+
+	SymlinkTypeMask = FlagDirectory | FlagSymlinkMissingTarget
 )
 
 const (
@@ -637,19 +641,3 @@ func (c *rawConnection) Statistics() Statistics {
 		OutBytesTotal: c.cw.Tot(),
 	}
 }
-
-func IsDeleted(bits uint32) bool {
-	return bits&FlagDeleted != 0
-}
-
-func IsInvalid(bits uint32) bool {
-	return bits&FlagInvalid != 0
-}
-
-func IsDirectory(bits uint32) bool {
-	return bits&FlagDirectory != 0
-}
-
-func HasPermissionBits(bits uint32) bool {
-	return bits&FlagNoPermBits == 0
-}

internal/scanner/blockqueue.go

@@ -68,7 +68,7 @@ func HashFile(path string, blockSize int) ([]protocol.BlockInfo, error) {
 
 func hashFiles(dir string, blockSize int, outbox, inbox chan protocol.FileInfo) {
 	for f := range inbox {
-		if protocol.IsDirectory(f.Flags) || protocol.IsDeleted(f.Flags) {
+		if f.IsDirectory() || f.IsDeleted() || f.IsSymlink() {
 			outbox <- f
 			continue
 		}

internal/scanner/blocks.go

@@ -129,3 +129,18 @@ func Verify(r io.Reader, blocksize int, blocks []protocol.BlockInfo) error {
 
 	return nil
 }
+
+// BlockEqual returns whether two slices of blocks are exactly the same hash
+// and index pair wise.
+func BlocksEqual(src, tgt []protocol.BlockInfo) bool {
+	if len(tgt) != len(src) {
+		return false
+	}
+
+	for i, sblk := range src {
+		if !bytes.Equal(sblk.Hash, tgt[i].Hash) {
+			return false
+		}
+	}
+	return true
+}

internal/scanner/walk.go

@@ -27,6 +27,7 @@ import (
 	"github.com/syncthing/syncthing/internal/ignore"
 	"github.com/syncthing/syncthing/internal/lamport"
 	"github.com/syncthing/syncthing/internal/protocol"
+	"github.com/syncthing/syncthing/internal/symlinks"
 )
 
 type Walker struct {
@@ -131,11 +132,75 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 			return nil
 		}
 
+		// We must perform this check, as symlinks on Windows are always
+		// .IsRegular or .IsDir unlike on Unix.
+		// Index wise symlinks are always files, regardless of what the target
+		// is, because symlinks carry their target path as their content.
+		isSymlink, _ := symlinks.IsSymlink(p)
+		if isSymlink {
+			var rval error
+			// If the target is a directory, do NOT descend down there.
+			// This will cause files to get tracked, and removing the symlink
+			// will as a result remove files in their real location.
+			// But do not SkipDir if the target is not a directory, as it will
+			// stop scanning the current directory.
+			if info.IsDir() {
+				rval = filepath.SkipDir
+			}
+
+			// We always rehash symlinks as they have no modtime or
+			// permissions.
+			// We check if they point to the old target by checking that
+			// their existing blocks match with the blocks in the index.
+			// If we don't have a filer or don't support symlinks, skip.
+			if w.CurrentFiler == nil || !symlinks.Supported {
+				return rval
+			}
+
+			target, flags, err := symlinks.Read(p)
+			flags = flags & protocol.SymlinkTypeMask
+			if err != nil {
+				if debug {
+					l.Debugln("readlink error:", p, err)
+				}
+				return rval
+			}
+
+			blocks, err := Blocks(strings.NewReader(target), w.BlockSize, 0)
+			if err != nil {
+				if debug {
+					l.Debugln("hash link error:", p, err)
+				}
+				return rval
+			}
+
+			cf := w.CurrentFiler.CurrentFile(rn)
+			if !cf.IsDeleted() && cf.IsSymlink() && SymlinkTypeEqual(flags, cf.Flags) && BlocksEqual(cf.Blocks, blocks) {
+				return rval
+			}
+
+			f := protocol.FileInfo{
+				Name:     rn,
+				Version:  lamport.Default.Tick(0),
+				Flags:    protocol.FlagSymlink | flags | protocol.FlagNoPermBits | 0666,
+				Modified: 0,
+				Blocks:   blocks,
+			}
+
+			if debug {
+				l.Debugln("symlink to hash:", p, f)
+			}
+
+			fchan <- f
+
+			return rval
+		}
+
 		if info.Mode().IsDir() {
 			if w.CurrentFiler != nil {
 				cf := w.CurrentFiler.CurrentFile(rn)
-				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
-				if !protocol.IsDeleted(cf.Flags) && protocol.IsDirectory(cf.Flags) && permUnchanged {
+				permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode()))
+				if !cf.IsDeleted() && cf.IsDirectory() && permUnchanged {
 					return nil
 				}
 			}
@@ -162,8 +227,8 @@ func (w *Walker) walkAndHashFiles(fchan chan protocol.FileInfo) filepath.WalkFun
 		if info.Mode().IsRegular() {
 			if w.CurrentFiler != nil {
 				cf := w.CurrentFiler.CurrentFile(rn)
-				permUnchanged := w.IgnorePerms || !protocol.HasPermissionBits(cf.Flags) || PermsEqual(cf.Flags, uint32(info.Mode()))
-				if !protocol.IsDeleted(cf.Flags) && cf.Modified == info.ModTime().Unix() && permUnchanged {
+				permUnchanged := w.IgnorePerms || !cf.HasPermissionBits() || PermsEqual(cf.Flags, uint32(info.Mode()))
+				if !cf.IsDeleted() && cf.Modified == info.ModTime().Unix() && permUnchanged {
 					return nil
 				}
 
@@ -215,3 +280,19 @@ func PermsEqual(a, b uint32) bool {
 		return a&0777 == b&0777
 	}
 }
+
+// If the target is missing, Unix never knows what type of symlink it is
+// and Windows always knows even if there is no target.
+// Which means that without this special check a Unix node would be fighting
+// with a Windows node about whether or not the target is known.
+// Basically, if you don't know and someone else knows, just accept it.
+// The fact that you don't know means you are on Unix, and on Unix you don't
+// really care what the target type is. The moment you do know, and if something
+// doesn't match, that will propogate throught the cluster.
+func SymlinkTypeEqual(disk, index uint32) bool {
+	if disk&protocol.FlagSymlinkMissingTarget != 0 && index&protocol.FlagSymlinkMissingTarget == 0 {
+		return true
+	}
+	return disk&protocol.SymlinkTypeMask == index&protocol.SymlinkTypeMask
+
+}

internal/symlinks/symlink_unix.go

@@ -0,0 +1,58 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+// +build !windows
+
+package symlinks
+
+import (
+	"os"
+
+	"github.com/syncthing/syncthing/internal/osutil"
+	"github.com/syncthing/syncthing/internal/protocol"
+)
+
+var (
+	Supported = true
+)
+
+func Read(path string) (string, uint32, error) {
+	var mode uint32
+	stat, err := os.Stat(path)
+	if err != nil {
+		mode = protocol.FlagSymlinkMissingTarget
+	} else if stat.IsDir() {
+		mode = protocol.FlagDirectory
+	}
+	path, err = os.Readlink(path)
+
+	return osutil.NormalizedFilename(path), mode, err
+}
+
+func IsSymlink(path string) (bool, error) {
+	lstat, err := os.Lstat(path)
+	if err != nil {
+		return false, err
+	}
+	return lstat.Mode()&os.ModeSymlink != 0, nil
+}
+
+func Create(source, target string, flags uint32) error {
+	return os.Symlink(osutil.NativeFilename(target), source)
+}
+
+func ChangeType(path string, flags uint32) error {
+	return nil
+}

internal/symlinks/symlink_windows.go

@@ -0,0 +1,203 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+// +build windows
+
+package symlinks
+
+import (
+	"os"
+	"path/filepath"
+
+	"github.com/syncthing/syncthing/internal/osutil"
+	"github.com/syncthing/syncthing/internal/protocol"
+
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const (
+	FSCTL_GET_REPARSE_POINT      = 0x900a8
+	FILE_FLAG_OPEN_REPARSE_POINT = 0x00200000
+	FILE_ATTRIBUTE_REPARSE_POINT = 0x400
+	IO_REPARSE_TAG_SYMLINK       = 0xA000000C
+	SYMBOLIC_LINK_FLAG_DIRECTORY = 0x1
+)
+
+var (
+	modkernel32            = syscall.NewLazyDLL("kernel32.dll")
+	procDeviceIoControl    = modkernel32.NewProc("DeviceIoControl")
+	procCreateSymbolicLink = modkernel32.NewProc("CreateSymbolicLinkW")
+
+	Supported = false
+)
+
+func init() {
+	// Needs administrator priviledges.
+	// Let's check that everything works.
+	// This could be done more officially:
+	// http://stackoverflow.com/questions/2094663/determine-if-windows-process-has-privilege-to-create-symbolic-link
+	// But I don't want to define 10 more structs just to look this up.
+	base := os.TempDir()
+	path := filepath.Join(base, "symlinktest")
+	defer os.Remove(path)
+
+	err := Create(path, base, protocol.FlagDirectory)
+	if err != nil {
+		return
+	}
+
+	isLink, err := IsSymlink(path)
+	if err != nil || !isLink {
+		return
+	}
+
+	target, flags, err := Read(path)
+	if err != nil || osutil.NativeFilename(target) != base || flags&protocol.FlagDirectory == 0 {
+		return
+	}
+	Supported = true
+}
+
+type reparseData struct {
+	reparseTag          uint32
+	reparseDataLength   uint16
+	reserved            uint16
+	substitueNameOffset uint16
+	substitueNameLength uint16
+	printNameOffset     uint16
+	printNameLength     uint16
+	flags               uint32
+	// substituteName - 264 widechars max = 528 bytes
+	// printName      - 260 widechars max = 520 bytes
+	//                                    = 1048 bytes total
+	buffer [1048]uint16
+}
+
+func (r *reparseData) PrintName() string {
+	// No clue why the offset and length is doubled...
+	offset := r.printNameOffset / 2
+	length := r.printNameLength / 2
+	return string(utf16.Decode(r.buffer[offset : offset+length]))
+}
+
+func (r *reparseData) SubstituteName() string {
+	// No clue why the offset and length is doubled...
+	offset := r.substitueNameOffset / 2
+	length := r.substitueNameLength / 2
+	return string(utf16.Decode(r.buffer[offset : offset+length]))
+}
+
+func Read(path string) (string, uint32, error) {
+	ptr, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return "", protocol.FlagSymlinkMissingTarget, err
+	}
+	handle, err := syscall.CreateFile(ptr, 0, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE|syscall.FILE_SHARE_DELETE, nil, syscall.OPEN_EXISTING, syscall.FILE_FLAG_BACKUP_SEMANTICS|FILE_FLAG_OPEN_REPARSE_POINT, 0)
+	if err != nil || handle == syscall.InvalidHandle {
+		return "", protocol.FlagSymlinkMissingTarget, err
+	}
+	defer syscall.Close(handle)
+	var ret uint16
+	var data reparseData
+
+	r1, _, err := syscall.Syscall9(procDeviceIoControl.Addr(), 8, uintptr(handle), FSCTL_GET_REPARSE_POINT, 0, 0, uintptr(unsafe.Pointer(&data)), unsafe.Sizeof(data), uintptr(unsafe.Pointer(&ret)), 0, 0)
+	if r1 == 0 {
+		return "", protocol.FlagSymlinkMissingTarget, err
+	}
+
+	var flags uint32 = 0
+	attr, err := syscall.GetFileAttributes(ptr)
+	if err != nil {
+		flags = protocol.FlagSymlinkMissingTarget
+	} else if attr&syscall.FILE_ATTRIBUTE_DIRECTORY != 0 {
+		flags = protocol.FlagDirectory
+	}
+
+	return osutil.NormalizedFilename(data.PrintName()), flags, nil
+}
+
+func IsSymlink(path string) (bool, error) {
+	ptr, err := syscall.UTF16PtrFromString(path)
+	if err != nil {
+		return false, err
+	}
+
+	attr, err := syscall.GetFileAttributes(ptr)
+	if err != nil {
+		return false, err
+	}
+	return attr&FILE_ATTRIBUTE_REPARSE_POINT != 0, nil
+}
+
+func Create(source, target string, flags uint32) error {
+	srcp, err := syscall.UTF16PtrFromString(source)
+	if err != nil {
+		return err
+	}
+
+	trgp, err := syscall.UTF16PtrFromString(osutil.NativeFilename(target))
+	if err != nil {
+		return err
+	}
+
+	// Sadly for Windows we need to specify the type of the symlink,
+	// whether it's a directory symlink or a file symlink.
+	// If the flags doesn't reveal the target type, try to evaluate it
+	// ourselves, and worst case default to the symlink pointing to a file.
+	mode := 0
+	if flags&protocol.FlagSymlinkMissingTarget != 0 {
+		path := target
+		if !filepath.IsAbs(target) {
+			path = filepath.Join(filepath.Dir(source), target)
+		}
+
+		stat, err := os.Stat(path)
+		if err == nil && stat.IsDir() {
+			mode = SYMBOLIC_LINK_FLAG_DIRECTORY
+		}
+	} else if flags&protocol.FlagDirectory != 0 {
+		mode = SYMBOLIC_LINK_FLAG_DIRECTORY
+	}
+
+	r0, _, err := syscall.Syscall(procCreateSymbolicLink.Addr(), 3, uintptr(unsafe.Pointer(srcp)), uintptr(unsafe.Pointer(trgp)), uintptr(mode))
+	if r0 == 1 {
+		return nil
+	}
+	return err
+}
+
+func ChangeType(path string, flags uint32) error {
+	target, cflags, err := Read(path)
+	if err != nil {
+		return err
+	}
+	// If it's the same type, nothing to do.
+	if cflags&protocol.SymlinkTypeMask == flags&protocol.SymlinkTypeMask {
+		return nil
+	}
+
+	// If the actual type is unknown, but the new type is file, nothing to do
+	if cflags&protocol.FlagSymlinkMissingTarget != 0 && flags&protocol.FlagDirectory == 0 {
+		return nil
+	}
+	return osutil.InWritableDir(func(path string) error {
+		// It should be a symlink as well hence no need to change permissions on
+		// the file.
+		os.Remove(path)
+		return Create(path, target, flags)
+	}, path)
+}

protocol/PROTOCOL.md

@@ -439,7 +439,7 @@ The Flags field is made up of the following single bit flags:
      0                   1                   2                   3
      0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-    |              Reserved           |P|I|D|   Unix Perm. & Mode   |
+    |              Reserved       |U|S|P|I|D|   Unix Perm. & Mode   |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
 
  - The lower 12 bits hold the common Unix permission and mode bits. An
@@ -461,7 +461,16 @@ The Flags field is made up of the following single bit flags:
    disregarded on files with this bit set. The permissions bits MUST be
    set to the octal value 0666.
 
- - Bit 0 through 16 are reserved for future use and SHALL be set to
+ - Bit 16 ("S") is set when the file is a symbolic link. The block list
+   SHALL be of one or more blocks since the target of the symlink is
+   stored within the blocks of the file.
+
+ - Bit 15 ("U") is set when the symbolic links target does not exist.
+   On systems where symbolic links have types, this bit being means
+   that the default file symlink SHALL be used. If this bit is unset
+   bit 19 will decide the type of symlink to be created.
+
+ - Bit 0 through 14 are reserved for future use and SHALL be set to
    zero.
 
 The hash algorithm is implied by the Hash length. Currently, the hash

test/common_test.go

@@ -30,6 +30,8 @@ import (
 	"os/exec"
 	"path/filepath"
 	"time"
+
+	"github.com/syncthing/syncthing/internal/symlinks"
 )
 
 func init() {
@@ -355,7 +357,22 @@ func startWalker(dir string, res chan<- fileInfo, abort <-chan struct{}) {
 		}
 
 		var f fileInfo
-		if info.IsDir() {
+		if ok, err := symlinks.IsSymlink(path); err == nil && ok {
+			f = fileInfo{
+				name: rn,
+				mode: os.ModeSymlink,
+			}
+
+			tgt, _, err := symlinks.Read(path)
+			if err != nil {
+				return err
+			}
+			h := md5.New()
+			h.Write([]byte(tgt))
+			hash := h.Sum(nil)
+
+			copy(f.hash[:], hash)
+		} else if info.IsDir() {
 			f = fileInfo{
 				name: rn,
 				mode: info.Mode(),

test/symlink_test.go

@@ -0,0 +1,280 @@
+// Copyright (C) 2014 The Syncthing Authors.
+//
+// This program is free software: you can redistribute it and/or modify it
+// under the terms of the GNU General Public License as published by the Free
+// Software Foundation, either version 3 of the License, or (at your option)
+// any later version.
+//
+// This program is distributed in the hope that it will be useful, but WITHOUT
+// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+// FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+// more details.
+//
+// You should have received a copy of the GNU General Public License along
+// with this program. If not, see <http://www.gnu.org/licenses/>.
+
+// +build integration
+
+package integration_test
+
+import (
+	"log"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/syncthing/syncthing/internal/symlinks"
+)
+
+func TestSymlinks(t *testing.T) {
+	log.Println("Cleaning...")
+	err := removeAll("s1", "s2", "h1/index", "h2/index")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Generating files...")
+	err = generateFiles("s1", 100, 20, "../bin/syncthing")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A file that we will replace with a symlink later
+
+	fd, err := os.Create("s1/fileToReplace")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fd.Close()
+
+	// A directory that we will replace with a symlink later
+
+	err = os.Mkdir("s1/dirToReplace", 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// A file and a symlink to that file
+
+	fd, err = os.Create("s1/file")
+	if err != nil {
+		t.Fatal(err)
+	}
+	fd.Close()
+	err = symlinks.Create("s1/fileLink", "file", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// A directory and a symlink to that directory
+
+	err = os.Mkdir("s1/dir", 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = symlinks.Create("s1/dirLink", "dir", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// A link to something in the repo that does not exist
+
+	err = symlinks.Create("s1/noneLink", "does/not/exist", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// A link we will replace with a file later
+
+	err = symlinks.Create("s1/repFileLink", "does/not/exist", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// A link we will replace with a directory later
+
+	err = symlinks.Create("s1/repDirLink", "does/not/exist", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Verify that the files and symlinks sync to the other side
+
+	log.Println("Syncing...")
+
+	sender := syncthingProcess{ // id1
+		log:    "1.out",
+		argv:   []string{"-home", "h1"},
+		port:   8081,
+		apiKey: apiKey,
+	}
+	err = sender.start()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	receiver := syncthingProcess{ // id2
+		log:    "2.out",
+		argv:   []string{"-home", "h2"},
+		port:   8082,
+		apiKey: apiKey,
+	}
+	err = receiver.start()
+	if err != nil {
+		sender.stop()
+		t.Fatal(err)
+	}
+
+	for {
+		comp, err := sender.peerCompletion()
+		if err != nil {
+			if strings.Contains(err.Error(), "use of closed network connection") {
+				time.Sleep(time.Second)
+				continue
+			}
+			sender.stop()
+			receiver.stop()
+			t.Fatal(err)
+		}
+
+		curComp := comp[id2]
+
+		if curComp == 100 {
+			sender.stop()
+			receiver.stop()
+			break
+		}
+
+		time.Sleep(time.Second)
+	}
+
+	sender.stop()
+	receiver.stop()
+
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	log.Println("Making some changes...")
+
+	// Remove one symlink
+
+	err = os.Remove("s1/fileLink")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Change the target of another
+
+	err = os.Remove("s1/dirLink")
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = symlinks.Create("s1/dirLink", "file", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// Replace one with a file
+
+	err = os.Remove("s1/repFileLink")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fd, err = os.Create("s1/repFileLink")
+	if err != nil {
+		log.Fatal(err)
+	}
+	fd.Close()
+
+	/* Currently fails, to be fixed with #80
+
+	// Replace one with a directory
+
+	err = os.Remove("s1/repDirLink")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	err = os.Mkdir("s1/repDirLink", 0755)
+	if err != nil {
+		log.Fatal(err)
+	}
+	*/
+
+	// Replace a file with a symlink
+
+	err = os.Remove("s1/fileToReplace")
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = symlinks.Create("s1/fileToReplace", "somewhere/non/existent", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	/* Currently fails, to be fixed with #80
+
+	// Replace a directory with a symlink
+
+	err = os.RemoveAll("s1/dirToReplace")
+	if err != nil {
+		log.Fatal(err)
+	}
+	err = symlinks.Create("s1/dirToReplace", "somewhere/non/existent", 0)
+	if err != nil {
+		log.Fatal(err)
+	}
+	*/
+
+	// Sync these changes and recheck
+
+	log.Println("Syncing...")
+
+	err = sender.start()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	err = receiver.start()
+	if err != nil {
+		sender.stop()
+		t.Fatal(err)
+	}
+
+	for {
+		comp, err := sender.peerCompletion()
+		if err != nil {
+			if strings.Contains(err.Error(), "use of closed network connection") {
+				time.Sleep(time.Second)
+				continue
+			}
+			sender.stop()
+			receiver.stop()
+			t.Fatal(err)
+		}
+
+		curComp := comp[id2]
+
+		if curComp == 100 {
+			sender.stop()
+			receiver.stop()
+			break
+		}
+
+		time.Sleep(time.Second)
+	}
+
+	sender.stop()
+	receiver.stop()
+
+	log.Println("Comparing directories...")
+	err = compareDirectories("s1", "s2")
+	if err != nil {
+		t.Fatal(err)
+	}
+}