SSH system commands: allow git and rsync inside virtual folders

dataprovider/user.go

@@ -64,9 +64,8 @@ var (
 // a denied file cannot be downloaded/overwritten/renamed but will still be
 // it will still be listed in the list of files.
 // System commands such as Git and rsync interacts with the filesystem directly
-// and they are not aware about these restrictions so rsync is not allowed if
-// extensions filters are defined and Git is not allowed inside a path with
-// extensions filters
+// and they are not aware about these restrictions so they are not allowed
+// inside paths with extensions filters
 type ExtensionsFilter struct {
 	// SFTP/SCP path, if no other specific filter is defined, the filter apply for
 	// sub directories too.
@@ -204,7 +203,7 @@ func (u *User) GetVirtualFolderForPath(sftpPath string) (vfs.VirtualFolder, erro
 	if len(u.VirtualFolders) == 0 || u.FsConfig.Provider != 0 {
 		return folder, errNoMatchingVirtualFolder
 	}
-	dirsForPath := utils.GetDirsForSFTPPath(path.Dir(sftpPath))
+	dirsForPath := utils.GetDirsForSFTPPath(sftpPath)
 	for _, val := range dirsForPath {
 		for _, v := range u.VirtualFolders {
 			if v.VirtualPath == val {
@@ -263,6 +262,9 @@ func (u *User) IsVirtualFolder(sftpPath string) bool {
 // HasVirtualFoldersInside return true if there are virtual folders inside the
 // specified SFTP path. We assume that path are cleaned
 func (u *User) HasVirtualFoldersInside(sftpPath string) bool {
+	if sftpPath == "/" && len(u.VirtualFolders) > 0 {
+		return true
+	}
 	for _, v := range u.VirtualFolders {
 		if len(v.VirtualPath) > len(sftpPath) {
 			if strings.HasPrefix(v.VirtualPath, sftpPath+"/") {
@@ -291,6 +293,24 @@ func (u *User) HasOverlappedMappedPaths() bool {
 	return false
 }
 
+// GetRemaingQuotaSize returns the available quota size for the given SFTP path
+func (u *User) GetRemaingQuotaSize(sftpPath string) int64 {
+	vfolder, err := u.GetVirtualFolderForPath(sftpPath)
+	if err == nil {
+		if vfolder.IsIncludedInUserQuota() && u.QuotaSize > 0 {
+			return u.QuotaSize - u.UsedQuotaSize
+		}
+		if vfolder.QuotaSize > 0 {
+			return vfolder.QuotaSize - vfolder.UsedQuotaSize
+		}
+	} else {
+		if u.QuotaSize > 0 {
+			return u.QuotaSize - u.UsedQuotaSize
+		}
+	}
+	return 0
+}
+
 // HasPerm returns true if the user has the given permission or any permission
 func (u *User) HasPerm(permission, path string) bool {
 	perms := u.GetPermissionsForPath(path)

docs/ssh-commands.md

@@ -1,16 +1,28 @@
 # SSH commands
 
-Some SSH commands are implemented directly inside SFTPGo, while for other commands we use system commands that need to be installed and in your system's `PATH`. For system commands we have no direct control on file creation/deletion and so we cannot support virtual folders, cloud storage filesystem, such as S3, and quota check is suboptimal. If quota is enabled, the number of files is checked at the command start and not while new files are created. The allowed size is calculated as the difference between the max quota and the used one, and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway, we see the bytes that the remote command sends to the local command via SSH. These bytes contain both protocol commands and files, and so the size of the files is different from the size trasferred via SSH: for example, a command can send compressed files, or a protocol command (few bytes) could delete a big file. To mitigate this issue, quotas are recalculated at the command end with a full home directory scan. This could be heavy for big directories. If you need system commands and quotas you could consider disabling quota restrictions and periodically update quota usage yourself using the REST API.
+Some SSH commands are implemented directly inside SFTPGo, while for others we use system commands that need to be installed and in your system's `PATH`.
 
-We support the following SSH commands:
+For system commands we have no direct control on file creation/deletion and so there are some limitations:
 
-- `scp`, we have our own SCP implementation since we can't rely on `scp` system command to proper handle quotas, user's home dir restrictions, cloud storage providers and virtual folders. SCP between two remote hosts is supported using the `-3` scp option.
-- `md5sum`, `sha1sum`, `sha256sum`, `sha384sum`, `sha512sum`. Useful to check message digests for uploaded files. These commands are implemented inside SFTPGo so they work even if the matching system commands are not available, for example, on Windows.
+- we cannot allow them if the target directory contains virtual folders or file extensions filters
+- system commands work only on local filyestem
+- we cannot avoid to leak real filesystem paths
+- quota check is suboptimal
+
+ If quota is enabled and SFTPGO receives a system command, the used size and number of files are checked at the command start and not while new files are created/deleted. While the command is running the number of files is not checked, the remaining size is calculated as the difference between the max allowed quota and the used one, and it is checked against the bytes transferred via SSH. The command is aborted if it uploads more bytes than the remaining allowed size calculated at the command start. Anyway, we only see the bytes that the remote command sends to the local one via SSH. These bytes contain both protocol commands and files, and so the size of the files is different from the size trasferred via SSH: for example, a command can send compressed files, or a protocol command (few bytes) could delete a big file. To mitigate these issues, quotas are recalculated at the command end with a full scan of the directory specified for the system command. This could be heavy for big directories. If you need system commands and quotas you could consider disabling quota restrictions and periodically update quota usage yourself using the REST API.
+
+ For these reasons we should limit system commands usage as much as possibile, we currently support the following system commands:
+
+- `git-receive-pack`, `git-upload-pack`, `git-upload-archive`. These commands enable support for Git repositories over SSH. They need to be installed and in your system's `PATH`.
+- `rsync`. The `rsync` command needs to be installed and in your system's `PATH`. We cannot avoid that rsync creates symlinks, so if the user has the permission to create symlinks, we add the option `--safe-links` to the received rsync command if it is not already set. This should prevent creating symlinks that point outside the home dir. If the user cannot create symlinks, we add the option `--munge-links` if it is not already set. This should make symlinks unusable (but manually recoverable).
+
+SFTPGo support the following built-in SSH commands:
+
+- `scp`, SFTPGo implements the SCP protocol so we can support it for cloud filesystems too and we can avoid the other system commands limitations. SCP between two remote hosts is supported using the `-3` scp option.
+- `md5sum`, `sha1sum`, `sha256sum`, `sha384sum`, `sha512sum`. Useful to check message digests for uploaded files.
 - `cd`, `pwd`. Some SFTP clients do not support the SFTP SSH_FXP_REALPATH packet type, so they use `cd` and `pwd` SSH commands to get the initial directory. Currently `cd` does nothing and `pwd` always returns the `/` path.
-- `git-receive-pack`, `git-upload-pack`, `git-upload-archive`. These commands enable support for Git repositories over SSH. They need to be installed and in your system's `PATH`. Git commands are not allowed inside virtual folders or inside directories with file extensions filters.
-- `rsync`. The `rsync` command needs to be installed and in your system's `PATH`. We cannot avoid that rsync creates symlinks, so if the user has the permission to create symlinks, we add the option `--safe-links` to the received rsync command if it is not already set. This should prevent creating symlinks that point outside the home dir. If the user cannot create symlinks, we add the option `--munge-links` if it is not already set. This should make symlinks unusable (but manually recoverable). The `rsync` command interacts with the filesystem directly and it is not aware of virtual folders and file extensions filters, so it will be automatically disabled for users with these features enabled.
-- `sftpgo-copy`. This is a builtin copy implementation. It allows server side copy for files and directories. The first argument is the source file/directory and the second one is the destination file/directory, for example `sftpgo-copy <src> <dst>`. The command will fail if the destination directory exists. Copy for directories spanning virtual folders is not supported.
-- `sftpgo-remove`. This is a builtin remove implementation. It allows to remove single files and to recursively remove directories. The first argument is the file/directory to remove, for example `sftpgo-remove <dst>`.
+- `sftpgo-copy`. This is a built-in copy implementation. It allows server side copy for files and directories. The first argument is the source file/directory and the second one is the destination file/directory, for example `sftpgo-copy <src> <dst>`. The command will fail if the destination directory exists. Copy for directories spanning virtual folders is not supported. Only local filesystem is supported: recursive copy for Cloud Storage filesystems requires a new request for every file in any case, so a server side copy is not possibile.
+- `sftpgo-remove`. This is a built-in remove implementation. It allows to remove single files and to recursively remove directories. The first argument is the file/directory to remove, for example `sftpgo-remove <dst>`. Only local filesystem is supported: recursive remove for Cloud Storage filesystems requires a new request for every file in any case, so a server side remove is not possibile.
 
 The following SSH commands are enabled by default:
 
@@ -18,4 +30,4 @@ The following SSH commands are enabled by default:
 - `sha1sum`
 - `cd`
 - `pwd`
-- `scp`
+- `scp`

sftpd/handler.go

@@ -480,7 +480,7 @@ func (c Connection) handleSFTPRemove(filePath string, request *sftp.Request) err
 
 	logger.CommandLog(removeLogSender, filePath, "", c.User.Username, "", c.ID, c.protocol, -1, -1, "", "", "")
 	if fi.Mode()&os.ModeSymlink != os.ModeSymlink {
-		vfolder, err := c.User.GetVirtualFolderForPath(request.Filepath)
+		vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(request.Filepath))
 		if err == nil {
 			dataprovider.UpdateVirtualFolderQuota(dataProvider, vfolder.BaseVirtualFolder, -1, -size, false) //nolint:errcheck
 			if vfolder.IsIncludedInUserQuota() {
@@ -573,7 +573,7 @@ func (c Connection) handleSFTPUploadToExistingFile(pflags sftp.FileOpenFlags, re
 		minWriteOffset = fileSize
 	} else {
 		if vfs.IsLocalOsFs(c.fs) {
-			vfolder, err := c.User.GetVirtualFolderForPath(requestPath)
+			vfolder, err := c.User.GetVirtualFolderForPath(path.Dir(requestPath))
 			if err == nil {
 				dataprovider.UpdateVirtualFolderQuota(dataProvider, vfolder.BaseVirtualFolder, 0, -fileSize, false) //nolint:errcheck
 				if vfolder.IsIncludedInUserQuota() {
@@ -619,8 +619,8 @@ func (c Connection) hasSpaceForRename(request *sftp.Request, initialSize int64,
 	if dataprovider.GetQuotaTracking() == 0 {
 		return true
 	}
-	sourceFolder, errSrc := c.User.GetVirtualFolderForPath(request.Filepath)
-	dstFolder, errDst := c.User.GetVirtualFolderForPath(request.Target)
+	sourceFolder, errSrc := c.User.GetVirtualFolderForPath(path.Dir(request.Filepath))
+	dstFolder, errDst := c.User.GetVirtualFolderForPath(path.Dir(request.Target))
 	if errSrc != nil && errDst != nil {
 		// rename inside the user home dir
 		return true
@@ -663,7 +663,7 @@ func (c Connection) hasSpace(checkFiles bool, requestPath string) bool {
 	var quotaFiles, numFiles int
 	var err error
 	var vfolder vfs.VirtualFolder
-	vfolder, err = c.User.GetVirtualFolderForPath(requestPath)
+	vfolder, err = c.User.GetVirtualFolderForPath(path.Dir(requestPath))
 	if err == nil && !vfolder.IsIncludedInUserQuota() {
 		if vfolder.HasNoQuotaRestrictions(checkFiles) {
 			return true
@@ -826,8 +826,8 @@ func (c Connection) updateQuotaAfterRename(request *sftp.Request, targetPath str
 	// - a file overwriting an existing one
 	// - a new directory
 	// initialSize != -1 only when overwriting files
-	sourceFolder, errSrc := c.User.GetVirtualFolderForPath(request.Filepath)
-	dstFolder, errDst := c.User.GetVirtualFolderForPath(request.Target)
+	sourceFolder, errSrc := c.User.GetVirtualFolderForPath(path.Dir(request.Filepath))
+	dstFolder, errDst := c.User.GetVirtualFolderForPath(path.Dir(request.Target))
 	if errSrc != nil && errDst != nil {
 		// both files are contained inside the user home dir
 		if initialSize != -1 {

sftpd/internal_test.go

@@ -939,48 +939,6 @@ func TestSSHCommandsRemoteFs(t *testing.T) {
 	assert.Error(t, err)
 }
 
-func TestSSHCommandQuotaScan(t *testing.T) {
-	buf := make([]byte, 65535)
-	stdErrBuf := make([]byte, 65535)
-	readErr := fmt.Errorf("test read error")
-	mockSSHChannel := MockChannel{
-		Buffer:       bytes.NewBuffer(buf),
-		StdErrBuffer: bytes.NewBuffer(stdErrBuf),
-		ReadError:    readErr,
-	}
-	server, client := net.Pipe()
-	defer func() {
-		err := server.Close()
-		assert.NoError(t, err)
-	}()
-	defer func() {
-		err := client.Close()
-		assert.NoError(t, err)
-	}()
-	permissions := make(map[string][]string)
-	permissions["/"] = []string{dataprovider.PermAny}
-	user := dataprovider.User{
-		Permissions: permissions,
-		QuotaFiles:  1,
-		HomeDir:     "invalid_path",
-	}
-	fs, err := user.GetFilesystem("123")
-	assert.NoError(t, err)
-	connection := Connection{
-		channel: &mockSSHChannel,
-		netConn: client,
-		User:    user,
-		fs:      fs,
-	}
-	cmd := sshCommand{
-		command:    "git-receive-pack",
-		connection: connection,
-		args:       []string{"/testrepo"},
-	}
-	err = cmd.rescanHomeDir()
-	assert.Error(t, err, "scanning an invalid home dir must fail")
-}
-
 func TestGitVirtualFolders(t *testing.T) {
 	permissions := make(map[string][]string)
 	permissions["/"] = []string{dataprovider.PermAny}
@@ -1006,7 +964,13 @@ func TestGitVirtualFolders(t *testing.T) {
 		VirtualPath: "/vdir",
 	})
 	_, err = cmd.getSystemCommand()
+	assert.NoError(t, err)
+	cmd.args = []string{"/"}
+	_, err = cmd.getSystemCommand()
 	assert.EqualError(t, err, errUnsupportedConfig.Error())
+	cmd.args = []string{"/vdir1"}
+	_, err = cmd.getSystemCommand()
+	assert.NoError(t, err)
 
 	cmd.connection.User.VirtualFolders = nil
 	cmd.connection.User.VirtualFolders = append(cmd.connection.User.VirtualFolders, vfs.VirtualFolder{
@@ -1017,7 +981,7 @@ func TestGitVirtualFolders(t *testing.T) {
 	})
 	cmd.args = []string{"/vdir/subdir"}
 	_, err = cmd.getSystemCommand()
-	assert.EqualError(t, err, errUnsupportedConfig.Error())
+	assert.NoError(t, err)
 
 	cmd.args = []string{"/adir/subdir"}
 	_, err = cmd.getSystemCommand()
@@ -1077,6 +1041,54 @@ func TestRsyncOptions(t *testing.T) {
 	assert.EqualError(t, err, errUnsupportedConfig.Error())
 }
 
+func TestSystemCommandSizeForPath(t *testing.T) {
+	permissions := make(map[string][]string)
+	permissions["/"] = []string{dataprovider.PermAny}
+	user := dataprovider.User{
+		Permissions: permissions,
+		HomeDir:     os.TempDir(),
+	}
+	fs, err := user.GetFilesystem("123")
+	assert.NoError(t, err)
+	conn := Connection{
+		User: user,
+		fs:   fs,
+	}
+	sshCmd := sshCommand{
+		command:    "rsync",
+		connection: conn,
+		args:       []string{"--server", "-vlogDtprze.iLsfxC", ".", "/"},
+	}
+	_, _, err = sshCmd.getSizeForPath("missing path")
+	assert.NoError(t, err)
+	testDir := filepath.Join(os.TempDir(), "dir")
+	err = os.MkdirAll(testDir, os.ModePerm)
+	assert.NoError(t, err)
+	testFile := filepath.Join(testDir, "testfile")
+	err = ioutil.WriteFile(testFile, []byte("test content"), os.ModePerm)
+	assert.NoError(t, err)
+	err = os.Symlink(testFile, testFile+".link")
+	assert.NoError(t, err)
+	numFiles, size, err := sshCmd.getSizeForPath(testFile + ".link")
+	assert.NoError(t, err)
+	assert.Equal(t, 0, numFiles)
+	assert.Equal(t, int64(0), size)
+	numFiles, size, err = sshCmd.getSizeForPath(testFile)
+	assert.NoError(t, err)
+	assert.Equal(t, 1, numFiles)
+	assert.Equal(t, int64(12), size)
+	if runtime.GOOS != osWindows {
+		err = os.Chmod(testDir, 0001)
+		assert.NoError(t, err)
+		_, _, err = sshCmd.getSizeForPath(testFile)
+		assert.Error(t, err)
+		err = os.Chmod(testDir, os.ModePerm)
+		assert.NoError(t, err)
+	}
+	err = os.RemoveAll(testDir)
+	assert.NoError(t, err)
+}
+
 func TestSystemCommandErrors(t *testing.T) {
 	buf := make([]byte, 65535)
 	stdErrBuf := make([]byte, 65535)
@@ -1099,9 +1111,14 @@ func TestSystemCommandErrors(t *testing.T) {
 	}()
 	permissions := make(map[string][]string)
 	permissions["/"] = []string{dataprovider.PermAny}
+	homeDir := filepath.Join(os.TempDir(), "adir")
+	err := os.MkdirAll(homeDir, os.ModePerm)
+	assert.NoError(t, err)
+	err = ioutil.WriteFile(filepath.Join(homeDir, "afile"), []byte("content"), os.ModePerm)
+	assert.NoError(t, err)
 	user := dataprovider.User{
 		Permissions: permissions,
-		HomeDir:     os.TempDir(),
+		HomeDir:     homeDir,
 	}
 	fs, err := user.GetFilesystem("123")
 	assert.NoError(t, err)
@@ -1111,16 +1128,25 @@ func TestSystemCommandErrors(t *testing.T) {
 		User:    user,
 		fs:      fs,
 	}
-	sshCmd := sshCommand{
-		command:    "ls",
-		connection: connection,
-		args:       []string{"/"},
+	var sshCmd sshCommand
+	if runtime.GOOS == osWindows {
+		sshCmd = sshCommand{
+			command:    "dir",
+			connection: connection,
+			args:       []string{"/"},
+		}
+	} else {
+		sshCmd = sshCommand{
+			command:    "ls",
+			connection: connection,
+			args:       []string{"/"},
+		}
 	}
 	systemCmd, err := sshCmd.getSystemCommand()
 	assert.NoError(t, err)
 
 	systemCmd.cmd.Dir = os.TempDir()
-	// FIXME: the command completes but the fake client was unable to read the response
+	// FIXME: the command completes but the fake client is unable to read the response
 	// no error is reported in this case. We can see that the expected code is executed
 	// reading the test coverage
 	sshCmd.executeSystemCommand(systemCmd) //nolint:errcheck
@@ -1160,6 +1186,10 @@ func TestSystemCommandErrors(t *testing.T) {
 	sshCmd.connection.channel = &mockSSHChannel
 	_, err = transfer.copyFromReaderToWriter(sshCmd.connection.channel, dst, 0)
 	assert.EqualError(t, err, io.ErrShortWrite.Error())
+	_, err = transfer.copyFromReaderToWriter(sshCmd.connection.channel, dst, -1)
+	assert.EqualError(t, err, errQuotaExceeded.Error())
+	err = os.RemoveAll(homeDir)
+	assert.NoError(t, err)
 }
 
 func TestTransferUpdateQuota(t *testing.T) {

sftpd/scp.go

@@ -195,10 +195,17 @@ func (c *scpCommand) handleUploadFile(resolvedPath, filePath string, sizeToRead
 		return err
 	}
 
+	file, w, cancelFn, err := c.connection.fs.Create(filePath, 0)
+	if err != nil {
+		c.connection.Log(logger.LevelError, logSenderSCP, "error creating file %#v: %v", resolvedPath, err)
+		c.sendErrorMessage(err)
+		return err
+	}
+
 	initialSize := int64(0)
 	if !isNewFile {
 		if vfs.IsLocalOsFs(c.connection.fs) {
-			vfolder, err := c.connection.User.GetVirtualFolderForPath(requestPath)
+			vfolder, err := c.connection.User.GetVirtualFolderForPath(path.Dir(requestPath))
 			if err == nil {
 				dataprovider.UpdateVirtualFolderQuota(dataProvider, vfolder.BaseVirtualFolder, 0, -fileSize, false) //nolint:errcheck
 				if vfolder.IsIncludedInUserQuota() {
@@ -211,12 +218,6 @@ func (c *scpCommand) handleUploadFile(resolvedPath, filePath string, sizeToRead
 			initialSize = fileSize
 		}
 	}
-	file, w, cancelFn, err := c.connection.fs.Create(filePath, 0)
-	if err != nil {
-		c.connection.Log(logger.LevelError, logSenderSCP, "error creating file %#v: %v", resolvedPath, err)
-		c.sendErrorMessage(err)
-		return err
-	}
 
 	vfs.SetPathPermissions(c.connection.fs, filePath, c.connection.User.GetUID(), c.connection.User.GetGID())
 

sftpd/sftpd_test.go

@@ -5147,9 +5147,8 @@ func TestGetVirtualFolderForPath(t *testing.T) {
 	assert.Equal(t, folder.MappedPath, mappedPath1)
 	_, err = user.GetVirtualFolderForPath("/vdir/sub1/file")
 	assert.Error(t, err)
-	// we check the parent dir
 	folder, err = user.GetVirtualFolderForPath(vdirPath)
-	assert.Error(t, err)
+	assert.NoError(t, err)
 }
 
 func TestSSHCommands(t *testing.T) {
@@ -5600,7 +5599,7 @@ func TestBasicGitCommands(t *testing.T) {
 	user, _, err := httpd.AddUser(u, http.StatusOK)
 	assert.NoError(t, err)
 
-	repoName := "testrepo"
+	repoName := "testrepo" //nolint:goconst
 	clonePath := filepath.Join(homeBasePath, repoName)
 	err = os.RemoveAll(user.GetHomeDir())
 	assert.NoError(t, err)
@@ -5624,17 +5623,25 @@ func TestBasicGitCommands(t *testing.T) {
 		printLatestLogs(10)
 	}
 
-	err = waitQuotaScans(1)
-	assert.NoError(t, err)
+	out, err = addFileToGitRepo(clonePath, 131072)
+	assert.NoError(t, err, "unexpected error, out: %v", string(out))
 
 	user, _, err = httpd.GetUserByID(user.ID, http.StatusOK)
 	assert.NoError(t, err)
-	user.QuotaSize = user.UsedQuotaSize - 1
+	user.QuotaSize = user.UsedQuotaSize + 1
 	_, _, err = httpd.UpdateUser(user, http.StatusOK)
 	assert.NoError(t, err)
 	out, err = pushToGitRepo(clonePath)
 	assert.Error(t, err, "git push must fail if quota is exceeded, out: %v", string(out))
 
+	aDir := filepath.Join(user.GetHomeDir(), repoName, "adir")
+	err = os.MkdirAll(aDir, 0001)
+	assert.NoError(t, err)
+	_, err = pushToGitRepo(clonePath)
+	assert.Error(t, err)
+	err = os.Chmod(aDir, os.ModePerm)
+	assert.NoError(t, err)
+
 	_, err = httpd.RemoveUser(user, http.StatusOK)
 	assert.NoError(t, err)
 
@@ -5644,6 +5651,73 @@ func TestBasicGitCommands(t *testing.T) {
 	assert.NoError(t, err)
 }
 
+func TestGitQuotaVirtualFolders(t *testing.T) {
+	if len(gitPath) == 0 || len(sshPath) == 0 || runtime.GOOS == osWindows {
+		t.Skip("git and/or ssh command not found or OS is windows, unable to execute this test")
+	}
+	usePubKey := true
+	repoName := "testrepo"
+	u := getTestUser(usePubKey)
+	u.QuotaFiles = 1
+	u.QuotaSize = 1
+	mappedPath := filepath.Join(os.TempDir(), "repo")
+	u.VirtualFolders = append(u.VirtualFolders, vfs.VirtualFolder{
+		BaseVirtualFolder: vfs.BaseVirtualFolder{
+			MappedPath: mappedPath,
+		},
+		VirtualPath: "/" + repoName,
+		QuotaFiles:  0,
+		QuotaSize:   0,
+	})
+	err := os.MkdirAll(mappedPath, os.ModePerm)
+	assert.NoError(t, err)
+	user, _, err := httpd.AddUser(u, http.StatusOK)
+	assert.NoError(t, err)
+	client, err := getSftpClient(user, usePubKey)
+	if assert.NoError(t, err) {
+		// we upload a file so the user is over quota
+		defer client.Close()
+		testFileName := "test_file.dat"
+		testFileSize := int64(131072)
+		testFilePath := filepath.Join(homeBasePath, testFileName)
+		err = createTestFile(testFilePath, testFileSize)
+		assert.NoError(t, err)
+		err = sftpUploadFile(testFilePath, testFileName, testFileSize, client)
+		assert.NoError(t, err)
+		err = os.Remove(testFilePath)
+		assert.NoError(t, err)
+	}
+
+	clonePath := filepath.Join(homeBasePath, repoName)
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+	err = os.RemoveAll(filepath.Join(homeBasePath, repoName))
+	assert.NoError(t, err)
+	out, err := initGitRepo(mappedPath)
+	assert.NoError(t, err, "unexpected error, out: %v", string(out))
+
+	out, err = cloneGitRepo(homeBasePath, "/"+repoName, user.Username)
+	assert.NoError(t, err, "unexpected error, out: %v", string(out))
+
+	out, err = addFileToGitRepo(clonePath, 128)
+	assert.NoError(t, err, "unexpected error, out: %v", string(out))
+
+	out, err = pushToGitRepo(clonePath)
+	assert.NoError(t, err, "unexpected error, out: %v", string(out))
+
+	_, err = httpd.RemoveUser(user, http.StatusOK)
+	assert.NoError(t, err)
+
+	err = os.RemoveAll(user.GetHomeDir())
+	assert.NoError(t, err)
+	_, err = httpd.RemoveFolder(vfs.BaseVirtualFolder{MappedPath: mappedPath}, http.StatusOK)
+	assert.NoError(t, err)
+	err = os.RemoveAll(mappedPath)
+	assert.NoError(t, err)
+	err = os.RemoveAll(clonePath)
+	assert.NoError(t, err)
+}
+
 func TestGitErrors(t *testing.T) {
 	if len(gitPath) == 0 || len(sshPath) == 0 || runtime.GOOS == osWindows {
 		t.Skip("git and/or ssh command not found or OS is windows, unable to execute this test")

sftpd/ssh_cmd.go

@@ -42,8 +42,9 @@ type sshCommand struct {
 }
 
 type systemCommand struct {
-	cmd      *exec.Cmd
-	realPath string
+	cmd            *exec.Cmd
+	fsPath         string
+	quotaCheckPath string
 }
 
 func processSSHCommand(payload []byte, connection *Connection, channel ssh.Channel, enabledSSHCommands []string) bool {
@@ -299,15 +300,21 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 	if !vfs.IsLocalOsFs(c.connection.fs) {
 		return c.sendErrorResponse(errUnsupportedConfig)
 	}
-	if c.connection.User.QuotaFiles > 0 && c.connection.User.UsedQuotaFiles > c.connection.User.QuotaFiles {
+	sshDestPath := c.getDestPath()
+	if !c.connection.hasSpace(true, command.quotaCheckPath) {
 		return c.sendErrorResponse(errQuotaExceeded)
 	}
 	perms := []string{dataprovider.PermDownload, dataprovider.PermUpload, dataprovider.PermCreateDirs, dataprovider.PermListItems,
-		dataprovider.PermOverwrite, dataprovider.PermDelete, dataprovider.PermRename}
-	if !c.connection.User.HasPerms(perms, c.getDestPath()) {
+		dataprovider.PermOverwrite, dataprovider.PermDelete}
+	if !c.connection.User.HasPerms(perms, sshDestPath) {
 		return c.sendErrorResponse(errPermissionDenied)
 	}
 
+	initialFiles, initialSize, err := c.getSizeForPath(command.fsPath)
+	if err != nil {
+		return c.sendErrorResponse(err)
+	}
+
 	stdin, err := command.cmd.StdinPipe()
 	if err != nil {
 		return c.sendErrorResponse(err)
@@ -337,13 +344,10 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 
 	go func() {
 		defer stdin.Close()
-		remainingQuotaSize := int64(0)
-		if c.connection.User.QuotaSize > 0 {
-			remainingQuotaSize = c.connection.User.QuotaSize - c.connection.User.UsedQuotaSize
-		}
+		remainingQuotaSize := c.connection.User.GetRemaingQuotaSize(sshDestPath)
 		transfer := Transfer{
 			file:           nil,
-			path:           command.realPath,
+			path:           command.fsPath,
 			start:          time.Now(),
 			bytesSent:      0,
 			bytesReceived:  0,
@@ -371,7 +375,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 	go func() {
 		transfer := Transfer{
 			file:           nil,
-			path:           command.realPath,
+			path:           command.fsPath,
 			start:          time.Now(),
 			bytesSent:      0,
 			bytesReceived:  0,
@@ -400,7 +404,7 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 	go func() {
 		transfer := Transfer{
 			file:           nil,
-			path:           command.realPath,
+			path:           command.fsPath,
 			start:          time.Now(),
 			bytesSent:      0,
 			bytesReceived:  0,
@@ -429,38 +433,48 @@ func (c *sshCommand) executeSystemCommand(command systemCommand) error {
 	<-commandResponse
 	err = command.cmd.Wait()
 	c.sendExitStatus(err)
-	c.rescanHomeDir() //nolint:errcheck
+
+	numFiles, dirSize, errSize := c.getSizeForPath(command.fsPath)
+	if errSize == nil {
+		c.updateQuota(sshDestPath, numFiles-initialFiles, dirSize-initialSize)
+	}
+	c.connection.Log(logger.LevelDebug, logSenderSSH, "command %#v finished for path %#v, initial files %v initial size %v "+
+		"current files %v current size %v size err: %v", c.connection.command, command.fsPath, initialFiles, initialSize,
+		numFiles, dirSize, errSize)
 	return err
 }
 
-func (c *sshCommand) checkGitAllowed() error {
-	gitPath := c.getDestPath()
-	for _, v := range c.connection.User.VirtualFolders {
-		if v.VirtualPath == gitPath {
-			c.connection.Log(logger.LevelDebug, logSenderSSH, "git is not supported inside virtual folder %#v user %#v",
-				gitPath, c.connection.User.Username)
-			return errUnsupportedConfig
-		}
-		if len(gitPath) > len(v.VirtualPath) {
-			if strings.HasPrefix(gitPath, v.VirtualPath+"/") {
-				c.connection.Log(logger.LevelDebug, logSenderSSH, "git is not supported inside virtual folder %#v user %#v",
-					gitPath, c.connection.User.Username)
-				return errUnsupportedConfig
-			}
-		}
+func (c *sshCommand) isSystemCommandAllowed() error {
+	sshDestPath := c.getDestPath()
+	if c.connection.User.IsVirtualFolder(sshDestPath) {
+		// overlapped virtual path are not allowed
+		return nil
+	}
+	if c.connection.User.HasVirtualFoldersInside(sshDestPath) {
+		c.connection.Log(logger.LevelDebug, logSenderSSH, "command %#v is not allowed, path %#v has virtual folders inside it, user %#v",
+			c.command, sshDestPath, c.connection.User.Username)
+		return errUnsupportedConfig
 	}
 	for _, f := range c.connection.User.Filters.FileExtensions {
-		if f.Path == gitPath {
+		if f.Path == sshDestPath {
 			c.connection.Log(logger.LevelDebug, logSenderSSH,
-				"git is not supported inside folder with files extensions filters %#v user %#v", gitPath,
-				c.connection.User.Username)
+				"command %#v is not allowed inside folders with files extensions filters %#v user %#v",
+				c.command, sshDestPath, c.connection.User.Username)
 			return errUnsupportedConfig
 		}
-		if len(gitPath) > len(f.Path) {
-			if strings.HasPrefix(gitPath, f.Path+"/") || f.Path == "/" {
+		if len(sshDestPath) > len(f.Path) {
+			if strings.HasPrefix(sshDestPath, f.Path+"/") || f.Path == "/" {
 				c.connection.Log(logger.LevelDebug, logSenderSSH,
-					"git is not supported inside folder with files extensions filters %#v user %#v", gitPath,
-					c.connection.User.Username)
+					"command %#v is not allowed it includes folders with files extensions filters %#v user %#v",
+					c.command, sshDestPath, c.connection.User.Username)
+				return errUnsupportedConfig
+			}
+		}
+		if len(sshDestPath) < len(f.Path) {
+			if strings.HasPrefix(sshDestPath+"/", f.Path) || sshDestPath == "/" {
+				c.connection.Log(logger.LevelDebug, logSenderSSH,
+					"command %#v is not allowed inside folder with files extensions filters %#v user %#v",
+					c.command, sshDestPath, c.connection.User.Username)
 				return errUnsupportedConfig
 			}
 		}
@@ -470,41 +484,34 @@ func (c *sshCommand) checkGitAllowed() error {
 
 func (c *sshCommand) getSystemCommand() (systemCommand, error) {
 	command := systemCommand{
-		cmd:      nil,
-		realPath: "",
+		cmd:            nil,
+		fsPath:         "",
+		quotaCheckPath: "",
 	}
 	args := make([]string, len(c.args))
 	copy(args, c.args)
-	var path string
+	var fsPath, quotaPath string
 	if len(c.args) > 0 {
 		var err error
 		sshPath := c.getDestPath()
-		path, err = c.connection.fs.ResolvePath(sshPath)
+		fsPath, err = c.connection.fs.ResolvePath(sshPath)
 		if err != nil {
 			return command, err
 		}
+		quotaPath = sshPath
+		fi, err := c.connection.fs.Stat(fsPath)
+		if err == nil && fi.IsDir() {
+			// if the target is an existing dir the command will write inside this dir
+			// so we need to check the quota for this directory and not its parent dir
+			quotaPath = path.Join(sshPath, "fakecontent")
+		}
 		args = args[:len(args)-1]
-		args = append(args, path)
+		args = append(args, fsPath)
 	}
-	if strings.HasPrefix(c.command, "git-") {
-		// we don't allow git inside virtual folders or folders with files extensions filters
-		if err := c.checkGitAllowed(); err != nil {
-			return command, err
-		}
+	if err := c.isSystemCommandAllowed(); err != nil {
+		return command, errUnsupportedConfig
 	}
 	if c.command == "rsync" {
-		// if the user has virtual folders or file extensions filters we don't allow rsync since the rsync command
-		// interacts with the filesystem directly and it is not aware about virtual folders/extensions files filters
-		if len(c.connection.User.VirtualFolders) > 0 {
-			c.connection.Log(logger.LevelDebug, logSenderSSH, "user %#v has virtual folders, rsync is not supported",
-				c.connection.User.Username)
-			return command, errUnsupportedConfig
-		}
-		if len(c.connection.User.Filters.FileExtensions) > 0 {
-			c.connection.Log(logger.LevelDebug, logSenderSSH, "user %#v has file extensions filter, rsync is not supported",
-				c.connection.User.Username)
-			return command, errUnsupportedConfig
-		}
 		// we cannot avoid that rsync creates symlinks so if the user has the permission
 		// to create symlinks we add the option --safe-links to the received rsync command if
 		// it is not already set. This should prevent to create symlinks that point outside
@@ -521,38 +528,18 @@ func (c *sshCommand) getSystemCommand() (systemCommand, error) {
 			}
 		}
 	}
-	c.connection.Log(logger.LevelDebug, logSenderSSH, "new system command %#v, with args: %v path: %v", c.command, args, path)
+	c.connection.Log(logger.LevelDebug, logSenderSSH, "new system command %#v, with args: %+v fs path %#v quota check path %#v",
+		c.command, args, fsPath, quotaPath)
 	cmd := exec.Command(c.command, args...)
 	uid := c.connection.User.GetUID()
 	gid := c.connection.User.GetGID()
 	cmd = wrapCmd(cmd, uid, gid)
 	command.cmd = cmd
-	command.realPath = path
+	command.fsPath = fsPath
+	command.quotaCheckPath = quotaPath
 	return command, nil
 }
 
-func (c *sshCommand) rescanHomeDir() error {
-	quotaTracking := dataprovider.GetQuotaTracking()
-	if (!c.connection.User.HasQuotaRestrictions() && quotaTracking == 2) || quotaTracking == 0 {
-		return nil
-	}
-	var err error
-	var numFiles int
-	var size int64
-	if AddQuotaScan(c.connection.User.Username) {
-		numFiles, size, err = c.connection.fs.ScanRootDirContents()
-		if err != nil {
-			c.connection.Log(logger.LevelWarn, logSenderSSH, "error scanning user home dir %#v: %v", c.connection.User.HomeDir, err)
-		} else {
-			err := dataprovider.UpdateUserQuota(dataProvider, c.connection.User, numFiles, size, true)
-			c.connection.Log(logger.LevelDebug, logSenderSSH, "user home dir scanned, user: %#v, dir: %#v, error: %v",
-				c.connection.User.Username, c.connection.User.HomeDir, err)
-		}
-		RemoveQuotaScan(c.connection.User.Username) //nolint:errcheck
-	}
-	return err
-}
-
 // for the supported commands, the destination path, if any, is the last argument
 func (c *sshCommand) getDestPath() string {
 	if len(c.args) == 0 {
@@ -642,6 +629,29 @@ func (c *sshCommand) checkCopyDestination(fsDestPath string) error {
 	return nil
 }
 
+func (c *sshCommand) getSizeForPath(name string) (int, int64, error) {
+	if dataprovider.GetQuotaTracking() > 0 {
+		fi, err := c.connection.fs.Lstat(name)
+		if err != nil {
+			if c.connection.fs.IsNotExist(err) {
+				return 0, 0, nil
+			}
+			c.connection.Log(logger.LevelDebug, logSenderSSH, "unable to stat %#v error: %v", name, err)
+			return 0, 0, err
+		}
+		if fi.IsDir() {
+			files, size, err := c.connection.fs.GetDirSize(name)
+			if err != nil {
+				c.connection.Log(logger.LevelDebug, logSenderSSH, "unable to get size for dir %#v error: %v", name, err)
+			}
+			return files, size, err
+		} else if fi.Mode().IsRegular() {
+			return 1, fi.Size(), nil
+		}
+	}
+	return 0, 0, nil
+}
+
 func (c *sshCommand) sendErrorResponse(err error) error {
 	errorString := fmt.Sprintf("%v: %v %v\n", c.command, c.getDestPath(), c.getMappedError(err))
 	c.connection.channel.Write([]byte(errorString)) //nolint:errcheck

sftpd/transfer.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"sync"
 	"time"
 
@@ -190,7 +191,7 @@ func (t *Transfer) updateQuota(numFiles int) bool {
 		return false
 	}
 	if t.transferType == transferUpload && (numFiles != 0 || t.bytesReceived > 0) {
-		vfolder, err := t.user.GetVirtualFolderForPath(t.requestPath)
+		vfolder, err := t.user.GetVirtualFolderForPath(path.Dir(t.requestPath))
 		if err == nil {
 			dataprovider.UpdateVirtualFolderQuota(dataProvider, vfolder.BaseVirtualFolder, numFiles, //nolint:errcheck
 				t.bytesReceived-t.initialSize, false)