gui, lib/config, lib/model: Allow absolute values for minimum disk free space (fixes #3307)

This deprecates the current minDiskFreePct setting and introduces
minDiskFree. The latter is, in it's serialized form, a string with a
unit. We accept percentages ("2.35%") and absolute values ("250 k", "12.5
Gi"). Common suffixes are understood. The config editor lets the user
enter the string, and validates it.

We still default to "1 %", but the user can change that to an absolute
value at will.

GitHub-Pull-Request: https://github.com/syncthing/syncthing/pull/4087
LGTM: AudriusButkevicius, imsodin

cmd/syncthing/main.go

@@ -1117,7 +1117,7 @@ func defaultConfig(myName string) config.Configuration {
 		defaultFolder = config.NewFolderConfiguration("default", locations[locDefFolder])
 		defaultFolder.Label = "Default Folder"
 		defaultFolder.RescanIntervalS = 60
-		defaultFolder.MinDiskFreePct = 1
+		defaultFolder.MinDiskFree = config.Size{Value: 1, Unit: "%"}
 		defaultFolder.Devices = []config.FolderDeviceConfiguration{{DeviceID: myID}}
 		defaultFolder.AutoNormalize = true
 		defaultFolder.MaxConflicts = -1

gui/default/assets/lang/lang-en.json

@@ -68,6 +68,7 @@
    "Editing {%path%}.": "Editing {{path}}.",
    "Enable NAT traversal": "Enable NAT traversal",
    "Enable Relaying": "Enable Relaying",
+   "Enter a non-negative number (e.g., \"2.35\") and select a unit. Percentages are as part of the total disk size.": "Enter a non-negative number (e.g., \"2.35\") and select a unit. Percentages are as part of the total disk size.",
    "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.": "Enter comma separated  (\"tcp://ip:port\", \"tcp://host:port\") addresses or \"dynamic\" to perform automatic discovery of the address.",
    "Enter ignore patterns, one per line.": "Enter ignore patterns, one per line.",
    "Error": "Error",
@@ -242,6 +243,7 @@
    "This Device": "This Device",
    "This can easily give hackers access to read and change any files on your computer.": "This can easily give hackers access to read and change any files on your computer.",
    "This is a major version upgrade.": "This is a major version upgrade.",
+   "This setting controls the free space required on the home (i.e., index database) disk.": "This setting controls the free space required on the home (i.e., index database) disk.",
    "Time": "Time",
    "Trash Can File Versioning": "Trash Can File Versioning",
    "Type": "Type",

gui/default/syncthing/core/syncthingController.js

@@ -62,7 +62,7 @@ angular.module('syncthing.core')
                 selectedDevices: {},
                 type: "readwrite",
                 rescanIntervalS: 60,
-                minDiskFreePct: 1,
+                minDiskFree: {value: 1, unit: "%"},
                 maxConflicts: 10,
                 fsync: true,
                 order: "random",

gui/default/syncthing/folder/editFolderModalView.html

@@ -65,19 +65,28 @@
       </div>
       <div id="folder-advanced" class="folder-advanced collapse">
         <div class="row">
-          <div class="col-md-12">
+          <div class="col-md-6">
             <div class="form-group" ng-class="{'has-error': folderEditor.rescanIntervalS.$invalid && folderEditor.rescanIntervalS.$dirty}">
-              <label for="rescanIntervalS"><span translate>Rescan Interval</span> (s)</label>
+              <label for="rescanIntervalS"><span translate>Rescan Interval</span> (s)</label><br/>
               <input name="rescanIntervalS" id="rescanIntervalS" class="form-control" type="number" ng-model="currentFolder.rescanIntervalS" required min="0">
               <p class="help-block">
                 <span translate ng-if="!folderEditor.rescanIntervalS.$valid && folderEditor.rescanIntervalS.$dirty">The rescan interval must be a non-negative number of seconds.</span>
               </p>
             </div>
-            <div class="form-group" ng-class="{'has-error': folderEditor.minDiskFreePct.$invalid && folderEditor.minDiskFreePct.$dirty}">
-              <label for="minDiskFreePct"><span translate>Minimum Free Disk Space</span> (0.0 - 100.0%)</label>
-              <input name="minDiskFreePct" id="minDiskFreePct" class="form-control" type="number" ng-model="currentFolder.minDiskFreePct" required min="0.0" max="100.0">
-              <p class="help-block">
-                <span translate ng-if="!folderEditor.minDiskFreePct.$valid && folderEditor.minDiskFreePct.$dirty">The minimum free disk space percentage must be a non-negative number between 0 and 100 (inclusive).</span>
+          </div>
+          <div class="col-md-6 form-horizontal">
+            <div class="form-group" ng-class="{'has-error': folderEditor.minDiskFree.$invalid && folderEditor.minDiskFree.$dirty}">
+              <label class="col-xs-12" for="minDiskFree"><span translate>Minimum Free Disk Space</span></label><br/>
+              <div class="col-xs-9"><input name="minDiskFree" id="minDiskFree" class="form-control" type="number" ng-model="currentFolder.minDiskFree.value" required min="0" step="0.01"></div>
+              <div class="col-xs-3"><select class="col-sm-3 form-control" ng-model="currentFolder.minDiskFree.unit">
+                <option value="%">%</option>
+                <option value="kB">kB</option>
+                <option value="MB">MB</option>
+                <option value="GB">GB</option>
+                <option value="TB">TB</option>
+              </select></div>
+              <p class="col-xs-12 help-block" ng-show="folderEditor.minDiskFree.$invalid">
+                <span translate>Enter a non-negative number (e.g., "2.35") and select a unit. Percentages are as part of the total disk size.</span>
               </p>
             </div>
           </div>

gui/default/syncthing/settings/settingsModalView.html

@@ -78,6 +78,25 @@
             <label translate for="GlobalAnnServersStr">Global Discovery Servers</label>
             <input ng-disabled="!tmpOptions.globalAnnounceEnabled" id="GlobalAnnServersStr" class="form-control" type="text" ng-model="tmpOptions._globalAnnounceServersStr">
           </div>
+
+          <div class="form-horizontal">
+            <div class="form-group" ng-class="{'has-error': settingsEditor.minHomeDiskFree.$invalid && settingsEditor.minHomeDiskFree.$dirty}">
+              <label class="col-xs-12" for="minHomeDiskFree"><span translate>Minimum Free Disk Space</span></label><br/>
+              <div class="col-xs-9"><input name="minHomeDiskFree" id="minHomeDiskFree" class="form-control" type="number" ng-model="tmpOptions.minHomeDiskFree.value" required min="0" step="0.01"></div>
+              <div class="col-xs-3"><select class="col-sm-3 form-control" ng-model="tmpOptions.minHomeDiskFree.unit">
+                <option value="%">%</option>
+                <option value="kB">kB</option>
+                <option value="MB">MB</option>
+                <option value="GB">GB</option>
+                <option value="TB">TB</option>
+              </select></div>
+              <p class="col-xs-12 help-block">
+                <span translate ng-show="settingsEditor.minHomeDiskFree.$invalid">Enter a non-negative number (e.g., "2.35") and select a unit. Percentages are as part of the total disk size.</span>
+                <span translate ng-hide="settingsEditor.minHomeDiskFree.$invalid">This setting controls the free space required on the home (i.e., index database) disk.</span>
+              </p>
+            </div>
+          </div>
+
         </div>
 
         <div class="col-md-6">

lib/config/config.go

@@ -29,7 +29,7 @@ import (
 
 const (
 	OldestHandledVersion = 10
-	CurrentVersion       = 19
+	CurrentVersion       = 20
 	MaxRescanIntervalS   = 365 * 24 * 60 * 60
 )
 
@@ -292,6 +292,9 @@ func (cfg *Configuration) clean() error {
 	if cfg.Version == 18 {
 		convertV18V19(cfg)
 	}
+	if cfg.Version == 19 {
+		convertV19V20(cfg)
+	}
 
 	// Build a list of available devices
 	existingDevices := make(map[protocol.DeviceID]bool)
@@ -341,6 +344,18 @@ func (cfg *Configuration) clean() error {
 	return nil
 }
 
+func convertV19V20(cfg *Configuration) {
+	cfg.Options.MinHomeDiskFree = Size{Value: cfg.Options.DeprecatedMinHomeDiskFreePct, Unit: "%"}
+	cfg.Options.DeprecatedMinHomeDiskFreePct = 0
+
+	for i := range cfg.Folders {
+		cfg.Folders[i].MinDiskFree = Size{Value: cfg.Folders[i].DeprecatedMinDiskFreePct, Unit: "%"}
+		cfg.Folders[i].DeprecatedMinDiskFreePct = 0
+	}
+
+	cfg.Version = 20
+}
+
 func convertV18V19(cfg *Configuration) {
 	// Triggers a database tweak
 	cfg.Version = 19
@@ -537,7 +552,7 @@ func convertV11V12(cfg *Configuration) {
 func convertV10V11(cfg *Configuration) {
 	// Set minimum disk free of existing folders to 1%
 	for i := range cfg.Folders {
-		cfg.Folders[i].MinDiskFreePct = 1
+		cfg.Folders[i].DeprecatedMinDiskFreePct = 1
 	}
 	cfg.Version = 11
 }

lib/config/config_test.go

@@ -55,7 +55,7 @@ func TestDefaultValues(t *testing.T) {
 		CacheIgnoredFiles:       false,
 		ProgressUpdateIntervalS: 5,
 		LimitBandwidthInLan:     false,
-		MinHomeDiskFreePct:      1,
+		MinHomeDiskFree:         Size{1, "%"},
 		URURL:                   "https://data.syncthing.net/newdata",
 		URInitialDelayS:         1800,
 		URPostInsecurely:        false,
@@ -110,7 +110,7 @@ func TestDeviceConfig(t *testing.T) {
 				Pullers:         0,
 				Hashers:         0,
 				AutoNormalize:   true,
-				MinDiskFreePct:  1,
+				MinDiskFree:     Size{1, "%"},
 				MaxConflicts:    -1,
 				Fsync:           true,
 				Versioning: VersioningConfiguration{
@@ -201,7 +201,7 @@ func TestOverriddenValues(t *testing.T) {
 		CacheIgnoredFiles:       true,
 		ProgressUpdateIntervalS: 10,
 		LimitBandwidthInLan:     true,
-		MinHomeDiskFreePct:      5.2,
+		MinHomeDiskFree:         Size{5.2, "%"},
 		URURL:                   "https://localhost/newdata",
 		URInitialDelayS:         800,
 		URPostInsecurely:        true,

lib/config/folderconfiguration.go

@@ -26,7 +26,7 @@ type FolderConfiguration struct {
 	RescanIntervalS       int                         `xml:"rescanIntervalS,attr" json:"rescanIntervalS"`
 	IgnorePerms           bool                        `xml:"ignorePerms,attr" json:"ignorePerms"`
 	AutoNormalize         bool                        `xml:"autoNormalize,attr" json:"autoNormalize"`
-	MinDiskFreePct        float64                     `xml:"minDiskFreePct" json:"minDiskFreePct"`
+	MinDiskFree           Size                        `xml:"minDiskFree" json:"minDiskFree"`
 	Versioning            VersioningConfiguration     `xml:"versioning" json:"versioning"`
 	Copiers               int                         `xml:"copiers" json:"copiers"` // This defines how many files are handled concurrently.
 	Pullers               int                         `xml:"pullers" json:"pullers"` // Defines how many blocks are fetched at the same time, possibly between separate copier routines.
@@ -45,7 +45,8 @@ type FolderConfiguration struct {
 
 	cachedPath string
 
-	DeprecatedReadOnly bool `xml:"ro,attr,omitempty" json:"-"`
+	DeprecatedReadOnly       bool    `xml:"ro,attr,omitempty" json:"-"`
+	DeprecatedMinDiskFreePct float64 `xml:"minDiskFreePct" json:"-"`
 }
 
 type FolderDeviceConfiguration struct {

lib/config/optionsconfiguration.go

@@ -123,7 +123,7 @@ type OptionsConfiguration struct {
 	CacheIgnoredFiles       bool                    `xml:"cacheIgnoredFiles" json:"cacheIgnoredFiles" default:"false"`
 	ProgressUpdateIntervalS int                     `xml:"progressUpdateIntervalS" json:"progressUpdateIntervalS" default:"5"`
 	LimitBandwidthInLan     bool                    `xml:"limitBandwidthInLan" json:"limitBandwidthInLan" default:"false"`
-	MinHomeDiskFreePct      float64                 `xml:"minHomeDiskFreePct" json:"minHomeDiskFreePct" default:"1"`
+	MinHomeDiskFree         Size                    `xml:"minHomeDiskFree" json:"minHomeDiskFree" default:"1 %"`
 	ReleasesURL             string                  `xml:"releasesURL" json:"releasesURL" default:"https://upgrades.syncthing.net/meta.json"`
 	AlwaysLocalNets         []string                `xml:"alwaysLocalNet" json:"alwaysLocalNets"`
 	OverwriteRemoteDevNames bool                    `xml:"overwriteRemoteDeviceNamesOnConnect" json:"overwriteRemoteDeviceNamesOnConnect" default:"false"`
@@ -141,11 +141,12 @@ type OptionsConfiguration struct {
 	KCPSendWindowSize       int                     `xml:"kcpSendWindowSize" json:"kcpSendWindowSize" default:"128"`
 	KCPReceiveWindowSize    int                     `xml:"kcpReceiveWindowSize" json:"kcpReceiveWindowSize" default:"128"`
 
-	DeprecatedUPnPEnabled  bool     `xml:"upnpEnabled,omitempty" json:"-"`
-	DeprecatedUPnPLeaseM   int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`
-	DeprecatedUPnPRenewalM int      `xml:"upnpRenewalMinutes,omitempty" json:"-"`
-	DeprecatedUPnPTimeoutS int      `xml:"upnpTimeoutSeconds,omitempty" json:"-"`
-	DeprecatedRelayServers []string `xml:"relayServer,omitempty" json:"-"`
+	DeprecatedUPnPEnabled        bool     `xml:"upnpEnabled,omitempty" json:"-"`
+	DeprecatedUPnPLeaseM         int      `xml:"upnpLeaseMinutes,omitempty" json:"-"`
+	DeprecatedUPnPRenewalM       int      `xml:"upnpRenewalMinutes,omitempty" json:"-"`
+	DeprecatedUPnPTimeoutS       int      `xml:"upnpTimeoutSeconds,omitempty" json:"-"`
+	DeprecatedRelayServers       []string `xml:"relayServer,omitempty" json:"-"`
+	DeprecatedMinHomeDiskFreePct float64  `xml:"minHomeDiskFreePct" json:"-"`
 }
 
 func (orig OptionsConfiguration) Copy() OptionsConfiguration {

lib/config/size.go

@@ -0,0 +1,75 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+type Size struct {
+	Value float64 `json:"value" xml:",chardata"`
+	Unit  string  `json:"unit" xml:"unit,attr"`
+}
+
+func ParseSize(s string) (Size, error) {
+	s = strings.TrimSpace(s)
+	if len(s) == 0 {
+		return Size{}, nil
+	}
+
+	var num, unit string
+	for i := 0; i < len(s) && (s[i] >= '0' && s[i] <= '9' || s[i] == '.' || s[i] == ','); i++ {
+		num = s[:i+1]
+	}
+	var i = len(num)
+	for i < len(s) && s[i] == ' ' {
+		i++
+	}
+	unit = s[i:]
+
+	val, err := strconv.ParseFloat(num, 64)
+	if err != nil {
+		return Size{}, err
+	}
+
+	return Size{val, unit}, nil
+}
+
+func (s Size) BaseValue() float64 {
+	unitPrefix := s.Unit
+	if len(unitPrefix) > 1 {
+		unitPrefix = unitPrefix[:1]
+	}
+
+	mult := 1.0
+	switch unitPrefix {
+	case "k", "K":
+		mult = 1000
+	case "m", "M":
+		mult = 1000 * 1000
+	case "g", "G":
+		mult = 1000 * 1000 * 1000
+	case "t", "T":
+		mult = 1000 * 1000 * 1000 * 1000
+	}
+
+	return s.Value * mult
+}
+
+func (s Size) Percentage() bool {
+	return strings.Contains(s.Unit, "%")
+}
+
+func (s Size) String() string {
+	return fmt.Sprintf("%v %s", s.Value, s.Unit)
+}
+
+func (Size) ParseDefault(s string) (interface{}, error) {
+	return ParseSize(s)
+}

lib/config/size_test.go

@@ -0,0 +1,72 @@
+// Copyright (C) 2017 The Syncthing Authors.
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this file,
+// You can obtain one at https://mozilla.org/MPL/2.0/.
+
+package config
+
+import "testing"
+
+func TestParseSize(t *testing.T) {
+	cases := []struct {
+		in  string
+		ok  bool
+		val float64
+		pct bool
+	}{
+		// We accept upper case SI units
+		{"5K", true, 5e3, false}, // even when they should be lower case
+		{"4 M", true, 4e6, false},
+		{"3G", true, 3e9, false},
+		{"2 T", true, 2e12, false},
+		// We accept lower case SI units out of user friendliness
+		{"1 k", true, 1e3, false},
+		{"2m", true, 2e6, false},
+		{"3 g", true, 3e9, false},
+		{"4t", true, 4e12, false},
+		// Fractions are OK
+		{"123.456 k", true, 123.456e3, false},
+		{"0.1234 m", true, 0.1234e6, false},
+		{"3.45 g", true, 3.45e9, false},
+		// We don't parse negative numbers
+		{"-1", false, 0, false},
+		{"-1k", false, 0, false},
+		{"-0.45g", false, 0, false},
+		// We accept various unit suffixes on the unit prefix
+		{"100 KBytes", true, 100e3, false},
+		{"100 Kbps", true, 100e3, false},
+		{"100 MAU", true, 100e6, false},
+		// Percentages are OK
+		{"1%", true, 1, true},
+		{"200%", true, 200, true},    // even large ones
+		{"200K%", true, 200e3, true}, // even with prefixes, although this makes no sense
+		{"2.34%", true, 2.34, true},  // fractions are A-ok
+		// The empty string is a valid zero
+		{"", true, 0, false},
+		{"  ", true, 0, false},
+	}
+
+	for _, tc := range cases {
+		size, err := ParseSize(tc.in)
+
+		if !tc.ok {
+			if err == nil {
+				t.Errorf("Unexpected nil error in UnmarshalText(%q)", tc.in)
+			}
+			continue
+		}
+
+		if err != nil {
+			t.Errorf("Unexpected error in UnmarshalText(%q): %v", tc.in, err)
+			continue
+		}
+		if size.BaseValue() > tc.val*1.001 || size.BaseValue() < tc.val*0.999 {
+			// Allow 0.1% slop due to floating point multiplication
+			t.Errorf("Incorrect value in UnmarshalText(%q): %v, wanted %v", tc.in, size.BaseValue(), tc.val)
+		}
+		if size.Percentage() != tc.pct {
+			t.Errorf("Incorrect percentage bool in UnmarshalText(%q): %v, wanted %v", tc.in, size.Percentage(), tc.pct)
+		}
+	}
+}

lib/config/testdata/v20.xml

@@ -0,0 +1,15 @@
+<configuration version="20">
+    <folder id="test" path="testdata" type="readonly" ignorePerms="false" rescanIntervalS="600" autoNormalize="true">
+        <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR"></device>
+        <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2"></device>
+        <minDiskFree unit="%">1</minDiskFree>
+        <maxConflicts>-1</maxConflicts>
+        <fsync>true</fsync>
+    </folder>
+    <device id="AIR6LPZ-7K4PTTV-UXQSMUU-CPQ5YWH-OEDFIIQ-JUG777G-2YQXXR5-YD6AWQR" name="node one" compression="metadata">
+        <address>tcp://a</address>
+    </device>
+    <device id="P56IOI7-MZJNU2Y-IQGDREY-DM2MGTI-MGL3BXN-PQ6W5BM-TBBZ4TJ-XZWICQ2" name="node two" compression="metadata">
+        <address>tcp://b</address>
+    </device>
+</configuration>

lib/model/model.go

@@ -109,8 +109,6 @@ var (
 	errFolderPathEmpty     = errors.New("folder path empty")
 	errFolderPathMissing   = errors.New("folder path missing")
 	errFolderMarkerMissing = errors.New("folder marker missing")
-	errHomeDiskNoSpace     = errors.New("home disk has insufficient free space")
-	errFolderNoSpace       = errors.New("folder has insufficient free space")
 	errInvalidFilename     = errors.New("filename is invalid")
 	errDeviceUnknown       = errors.New("unknown device")
 	errDevicePaused        = errors.New("device is paused")
@@ -2298,29 +2296,31 @@ func (m *Model) checkFolderPath(folder config.FolderConfiguration) error {
 // checkFolderFreeSpace returns nil if the folder has the required amount of
 // free space, or if folder free space checking is disabled.
 func (m *Model) checkFolderFreeSpace(folder config.FolderConfiguration) error {
-	if folder.MinDiskFreePct <= 0 {
-		return nil
-	}
-
-	free, err := osutil.DiskFreePercentage(folder.Path())
-	if err == nil && free < folder.MinDiskFreePct {
-		return errFolderNoSpace
-	}
-
-	return nil
+	return m.checkFreeSpace(folder.MinDiskFree, folder.Path())
 }
 
 // checkHomeDiskFree returns nil if the home disk has the required amount of
 // free space, or if home disk free space checking is disabled.
 func (m *Model) checkHomeDiskFree() error {
-	minFree := m.cfg.Options().MinHomeDiskFreePct
-	if minFree <= 0 {
+	return m.checkFreeSpace(m.cfg.Options().MinHomeDiskFree, m.cfg.ConfigPath())
+}
+
+func (m *Model) checkFreeSpace(req config.Size, path string) error {
+	val := req.BaseValue()
+	if val <= 0 {
 		return nil
 	}
 
-	free, err := osutil.DiskFreePercentage(m.cfg.ConfigPath())
-	if err == nil && free < minFree {
-		return errHomeDiskNoSpace
+	if req.Percentage() {
+		free, err := osutil.DiskFreePercentage(path)
+		if err == nil && free < val {
+			return fmt.Errorf("insufficient space in %v: %f %% < %v", path, free, req)
+		}
+	} else {
+		free, err := osutil.DiskFreeBytes(path)
+		if err == nil && float64(free) < val {
+			return fmt.Errorf("insufficient space in %v: %v < %v", path, free, req)
+		}
 	}
 
 	return nil

lib/model/rwfolder.go

@@ -1110,7 +1110,7 @@ func (f *sendReceiveFolder) handleFile(file protocol.FileInfo, copyChan chan<- c
 		blocksSize = file.Size
 	}
 
-	if f.MinDiskFreePct > 0 {
+	if f.MinDiskFree.BaseValue() > 0 {
 		if free, err := osutil.DiskFreeBytes(f.dir); err == nil && free < blocksSize {
 			l.Warnf(`Folder "%s": insufficient disk space in %s for %s: have %.2f MiB, need %.2f MiB`, f.folderID, f.dir, file.Name, float64(free)/1024/1024, float64(blocksSize)/1024/1024)
 			f.newError(file.Name, errors.New("insufficient space"))