Remove legacy data retention API

Data retention is now managed via the EventManager, introduced in v2.4.0.
This allows scheduling retention checks and sending email or HTTP notifications,
making the old API redundant.

Signed-off-by: Nicola Murino <[email protected]>

examples/data-retention/README.md

@@ -1,36 +0,0 @@
-# File retention policies
-
-:warning: Since v2.4.0 you can use the [EventManager](https://docs.sftpgo.com/latest/eventmanager/) to schedule data retention checks.
-
-The `checkretention` example script shows how to use the SFTPGo REST API to manage data retention.
-
-:warning: Deleting files is an irreversible action, please make sure you fully understand what you are doing before using this feature, you may have users with overlapping home directories or virtual folders shared between multiple users, it is relatively easy to inadvertently delete files you need.
-
-The example shows how to setup a really simple retention policy, for each user it sends this request:
-
-```json
-[
-  {
-    "path": "/",
-    "retention": 168,
-    "delete_empty_dirs": true,
-    "ignore_user_permissions": false
-  }
-]
-```
-
-so alls files with modification time older than 168 hours (7 days) will be deleted. Empty directories will be removed and the check will respect user's permissions, so if the user cannot delete a file/folder it will be skipped.
-
-You can define different retention policies per-user and per-folder and you can exclude a folder setting the retention to `0`.
-
-You can use this script as a starting point, please edit it according to your needs.
-
-The script is written in Python and has the following requirements:
-
-- python3 or python2
-- python [Requests](https://requests.readthedocs.io/en/master/) module
-
-The provided example tries to connect to an SFTPGo instance running on `127.0.0.1:8080` using the following credentials:
-
-- username: `admin`
-- password: `password`

examples/data-retention/checkretention

@@ -1,115 +0,0 @@
-#!/usr/bin/env python
-
-from datetime import datetime
-import sys
-import time
-
-import pytz
-import requests
-
-try:
-	import urllib.parse as urlparse
-except ImportError:
-	import urlparse
-
-# change base_url to point to your SFTPGo installation
-base_url = "http://127.0.0.1:8080"
-# set to False if you want to skip TLS certificate validation
-verify_tls_cert = True
-# set the credentials for a valid admin here
-admin_user = "admin"
-admin_password = "password"
-
-
-class CheckRetention:
-
-	def __init__(self):
-		self.limit = 100
-		self.offset = 0
-		self.access_token = ""
-		self.access_token_expiration = None
-
-	def printLog(self, message):
-		print("{} - {}".format(datetime.now(), message))
-
-	def checkAccessToken(self):
-		if self.access_token != "" and self.access_token_expiration:
-			expire_diff = self.access_token_expiration - datetime.now(tz=pytz.UTC)
-			# we don't use total_seconds to be python 2 compatible
-			seconds_to_expire = expire_diff.days * 86400 + expire_diff.seconds
-			if seconds_to_expire > 180:
-				return
-
-		auth = requests.auth.HTTPBasicAuth(admin_user, admin_password)
-		r = requests.get(urlparse.urljoin(base_url, "api/v2/token"), auth=auth, verify=verify_tls_cert, timeout=10)
-		if r.status_code != 200:
-			self.printLog("error getting access token: {}".format(r.text))
-			sys.exit(1)
-		self.access_token = r.json()["access_token"]
-		self.access_token_expiration = pytz.timezone("UTC").localize(datetime.strptime(r.json()["expires_at"],
-																					"%Y-%m-%dT%H:%M:%SZ"))
-
-	def getAuthHeader(self):
-		self.checkAccessToken()
-		return {"Authorization": "Bearer " + self.access_token}
-
-	def waitForRentionCheck(self, username):
-		while True:
-			auth_header = self.getAuthHeader()
-			r = requests.get(urlparse.urljoin(base_url, "api/v2/retention/users/checks"), headers=auth_header, verify=verify_tls_cert,
-							timeout=10)
-			if r.status_code != 200:
-				self.printLog("error getting retention checks while waiting for {}: {}".format(username, r.text))
-				sys.exit(1)
-
-			checking = False
-			for check in r.json():
-				if check["username"] == username:
-					checking = True
-			if not checking:
-				break
-			self.printLog("waiting for the retention check to complete for user {}".format(username))
-			time.sleep(2)
-
-		self.printLog("retention check for user {} finished".format(username))
-
-	def checkUserRetention(self, username):
-		self.printLog("starting retention check for user {}".format(username))
-		auth_header = self.getAuthHeader()
-		retention = [
-						{
-							"path": "/",
-							"retention": 168,
-							"delete_empty_dirs": True,
-							"ignore_user_permissions": False
-						}
-					]
-		r = requests.post(urlparse.urljoin(base_url, "api/v2/retention/users/" + username + "/check"), headers=auth_header,
-						 json=retention, verify=verify_tls_cert, timeout=10)
-		if r.status_code != 202:
-			self.printLog("error starting retention check for user {}: {}".format(username, r.text))
-			sys.exit(1)
-		self.waitForRentionCheck(username)
-
-	def checkUsersRetention(self):
-		while True:
-			self.printLog("get users, limit {} offset {}".format(self.limit, self.offset))
-			auth_header = self.getAuthHeader()
-			payload = {"limit":self.limit, "offset":self.offset}
-			r = requests.get(urlparse.urljoin(base_url, "api/v2/users"), headers=auth_header, params=payload,
-						verify=verify_tls_cert, timeout=10)
-			if r.status_code != 200:
-				self.printLog("error getting users: {}".format(r.text))
-				sys.exit(1)
-			users = r.json()
-			for user in users:
-				self.checkUserRetention(user["username"])
-
-			self.offset += len(users)
-			if len(users) < self.limit:
-				break
-
-
-if __name__ == '__main__':
-	c = CheckRetention()
-	c.checkUsersRetention()

internal/common/dataretention.go

@@ -15,44 +15,20 @@
 package common
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
-	"net/url"
 	"os"
-	"os/exec"
 	"path"
-	"path/filepath"
-	"strings"
 	"sync"
 	"time"
 
-	"github.com/wneessen/go-mail"
-
-	"github.com/drakkan/sftpgo/v2/internal/command"
 	"github.com/drakkan/sftpgo/v2/internal/dataprovider"
-	"github.com/drakkan/sftpgo/v2/internal/httpclient"
 	"github.com/drakkan/sftpgo/v2/internal/logger"
-	"github.com/drakkan/sftpgo/v2/internal/smtp"
 	"github.com/drakkan/sftpgo/v2/internal/util"
 	"github.com/drakkan/sftpgo/v2/internal/vfs"
 )
 
-// RetentionCheckNotification defines the supported notification methods for a retention check result
-type RetentionCheckNotification = string
-
-// Supported notification methods
-const (
-	// notify results using the defined "data_retention_hook"
-	RetentionCheckNotificationHook = "Hook"
-	// notify results by email
-	RetentionCheckNotificationEmail = "Email"
-)
-
 var (
 	// RetentionChecks is the list of active retention checks
 	RetentionChecks ActiveRetentionChecks
@@ -74,14 +50,10 @@ func (c *ActiveRetentionChecks) Get(role string) []RetentionCheck {
 		if role == "" || role == check.Role {
 			foldersCopy := make([]dataprovider.FolderRetention, len(check.Folders))
 			copy(foldersCopy, check.Folders)
-			notificationsCopy := make([]string, len(check.Notifications))
-			copy(notificationsCopy, check.Notifications)
 			checks = append(checks, RetentionCheck{
-				Username:      check.Username,
-				StartTime:     check.StartTime,
-				Notifications: notificationsCopy,
-				Email:         check.Email,
-				Folders:       foldersCopy,
+				Username:  check.Username,
+				StartTime: check.StartTime,
+				Folders:   foldersCopy,
 			})
 		}
 	}
@@ -150,54 +122,10 @@ type RetentionCheck struct {
 	StartTime int64 `json:"start_time"`
 	// affected folders
 	Folders []dataprovider.FolderRetention `json:"folders"`
-	// how cleanup results will be notified
-	Notifications []RetentionCheckNotification `json:"notifications,omitempty"`
-	// email to use if the notification method is set to email
-	Email string `json:"email,omitempty"`
-	Role  string `json:"-"`
+	Role    string                         `json:"-"`
 	// Cleanup results
 	results []folderRetentionCheckResult `json:"-"`
-	conn    *BaseConnection
-}
-
-// Validate returns an error if the specified folders are not valid
-func (c *RetentionCheck) Validate() error {
-	folderPaths := make(map[string]bool)
-	nothingToDo := true
-	for idx := range c.Folders {
-		f := &c.Folders[idx]
-		if err := f.Validate(); err != nil {
-			return err
-		}
-		if f.Retention > 0 {
-			nothingToDo = false
-		}
-		if _, ok := folderPaths[f.Path]; ok {
-			return util.NewValidationError(fmt.Sprintf("duplicated folder path %q", f.Path))
-		}
-		folderPaths[f.Path] = true
-	}
-	if nothingToDo {
-		return util.NewValidationError("nothing to delete!")
-	}
-	for _, notification := range c.Notifications {
-		switch notification {
-		case RetentionCheckNotificationEmail:
-			if !smtp.IsEnabled() {
-				return util.NewValidationError("in order to notify results via email you must configure an SMTP server")
-			}
-			if c.Email == "" {
-				return util.NewValidationError("in order to notify results via email you must add a valid email address to your profile")
-			}
-		case RetentionCheckNotificationHook:
-			if Config.DataRetentionHook == "" {
-				return util.NewValidationError("in order to notify results via hook you must define a data_retention_hook")
-			}
-		default:
-			return util.NewValidationError(fmt.Sprintf("invalid notification %q", notification))
-		}
-	}
-	return nil
+	conn    *BaseConnection              `json:"-"`
 }
 
 func (c *RetentionCheck) updateUserPermissions() {
@@ -359,130 +287,13 @@ func (c *RetentionCheck) Start() error {
 	for _, folder := range c.Folders {
 		if folder.Retention > 0 {
 			if err := c.cleanupFolder(folder.Path, 0); err != nil {
-				c.conn.Log(logger.LevelError, "retention check failed, unable to cleanup folder %q", folder.Path)
-				c.sendNotifications(time.Since(startTime), err)
+				c.conn.Log(logger.LevelError, "retention check failed, unable to cleanup folder %q, elapsed: %s",
+					folder.Path, time.Since(startTime))
 				return err
 			}
 		}
 	}
 
-	c.conn.Log(logger.LevelInfo, "retention check completed")
-	c.sendNotifications(time.Since(startTime), nil)
-	return nil
-}
-
-func (c *RetentionCheck) sendNotifications(elapsed time.Duration, err error) {
-	for _, notification := range c.Notifications {
-		switch notification {
-		case RetentionCheckNotificationEmail:
-			c.sendEmailNotification(err) //nolint:errcheck
-		case RetentionCheckNotificationHook:
-			c.sendHookNotification(elapsed, err) //nolint:errcheck
-		}
-	}
-}
-
-func (c *RetentionCheck) sendEmailNotification(errCheck error) error {
-	params := EventParams{}
-	if len(c.results) > 0 || errCheck != nil {
-		params.retentionChecks = append(params.retentionChecks, executedRetentionCheck{
-			Username:   c.conn.User.Username,
-			ActionName: "Retention check",
-			Results:    c.results,
-		})
-	}
-	var files []*mail.File
-	f, err := params.getRetentionReportsAsMailAttachment()
-	if err != nil {
-		c.conn.Log(logger.LevelError, "unable to get retention report as mail attachment: %v", err)
-		return err
-	}
-	f.Name = "retention-report.zip"
-	files = append(files, f)
-
-	startTime := time.Now()
-	var subject string
-	if errCheck == nil {
-		subject = fmt.Sprintf("Successful retention check for user %q", c.conn.User.Username)
-	} else {
-		subject = fmt.Sprintf("Retention check failed for user %q", c.conn.User.Username)
-	}
-	body := "Further details attached."
-	err = smtp.SendEmail([]string{c.Email}, nil, subject, body, smtp.EmailContentTypeTextPlain, files...)
-	if err != nil {
-		c.conn.Log(logger.LevelError, "unable to notify retention check result via email: %v, elapsed: %s", err,
-			time.Since(startTime))
-		return err
-	}
-	c.conn.Log(logger.LevelInfo, "retention check result successfully notified via email, elapsed: %s", time.Since(startTime))
+	c.conn.Log(logger.LevelInfo, "retention check completed, elapsed: %s", time.Since(startTime))
 	return nil
 }
-
-func (c *RetentionCheck) sendHookNotification(elapsed time.Duration, errCheck error) error {
-	startNewHook()
-	defer hookEnded()
-
-	data := make(map[string]any)
-	totalDeletedFiles := 0
-	totalDeletedSize := int64(0)
-	for _, result := range c.results {
-		totalDeletedFiles += result.DeletedFiles
-		totalDeletedSize += result.DeletedSize
-	}
-	data["username"] = c.conn.User.Username
-	data["start_time"] = c.StartTime
-	data["elapsed"] = elapsed.Milliseconds()
-	if errCheck == nil {
-		data["status"] = 1
-	} else {
-		data["status"] = 0
-	}
-	data["total_deleted_files"] = totalDeletedFiles
-	data["total_deleted_size"] = totalDeletedSize
-	data["details"] = c.results
-	jsonData, _ := json.Marshal(data)
-
-	startTime := time.Now()
-
-	if strings.HasPrefix(Config.DataRetentionHook, "http") {
-		var url *url.URL
-		url, err := url.Parse(Config.DataRetentionHook)
-		if err != nil {
-			c.conn.Log(logger.LevelError, "invalid data retention hook %q: %v", Config.DataRetentionHook, err)
-			return err
-		}
-		respCode := 0
-
-		resp, err := httpclient.RetryablePost(url.String(), "application/json", bytes.NewBuffer(jsonData))
-		if err == nil {
-			respCode = resp.StatusCode
-			resp.Body.Close()
-
-			if respCode != http.StatusOK {
-				err = errUnexpectedHTTResponse
-			}
-		}
-
-		c.conn.Log(logger.LevelDebug, "notified result to URL: %q, status code: %v, elapsed: %v err: %v",
-			url.Redacted(), respCode, time.Since(startTime), err)
-
-		return err
-	}
-	if !filepath.IsAbs(Config.DataRetentionHook) {
-		err := fmt.Errorf("invalid data retention hook %q", Config.DataRetentionHook)
-		c.conn.Log(logger.LevelError, "%v", err)
-		return err
-	}
-	timeout, env, args := command.GetConfig(Config.DataRetentionHook, command.HookDataRetention)
-	ctx, cancel := context.WithTimeout(context.Background(), timeout)
-	defer cancel()
-
-	cmd := exec.CommandContext(ctx, Config.DataRetentionHook, args...)
-	cmd.Env = append(env,
-		fmt.Sprintf("SFTPGO_DATA_RETENTION_RESULT=%s", jsonData))
-	err := cmd.Run()
-
-	c.conn.Log(logger.LevelDebug, "notified result using command: %q, elapsed: %s err: %v",
-		Config.DataRetentionHook, time.Since(startTime), err)
-	return err
-}

internal/common/dataretention_test.go

@@ -15,228 +15,17 @@
 package common
 
 import (
-	"errors"
 	"fmt"
-	"os/exec"
-	"runtime"
 	"testing"
-	"time"
 
 	"github.com/sftpgo/sdk"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"github.com/drakkan/sftpgo/v2/internal/dataprovider"
-	"github.com/drakkan/sftpgo/v2/internal/smtp"
 	"github.com/drakkan/sftpgo/v2/internal/util"
 )
 
-func TestRetentionValidation(t *testing.T) {
-	check := RetentionCheck{}
-	check.Folders = []dataprovider.FolderRetention{
-		{
-			Path:      "/",
-			Retention: -1,
-		},
-	}
-	err := check.Validate()
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "invalid folder retention")
-
-	check.Folders = []dataprovider.FolderRetention{
-		{
-			Path:      "/ab/..",
-			Retention: 0,
-		},
-	}
-	err = check.Validate()
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "nothing to delete")
-	assert.Equal(t, "/", check.Folders[0].Path)
-
-	check.Folders = append(check.Folders, dataprovider.FolderRetention{
-		Path:      "/../..",
-		Retention: 24,
-	})
-	err = check.Validate()
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), `duplicated folder path "/"`)
-
-	check.Folders = []dataprovider.FolderRetention{
-		{
-			Path:      "/dir1",
-			Retention: 48,
-		},
-		{
-			Path:      "/dir2",
-			Retention: 96,
-		},
-	}
-	err = check.Validate()
-	assert.NoError(t, err)
-	assert.Len(t, check.Notifications, 0)
-	assert.Empty(t, check.Email)
-
-	check.Notifications = []RetentionCheckNotification{RetentionCheckNotificationEmail}
-	err = check.Validate()
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "you must configure an SMTP server")
-
-	smtpCfg := smtp.Config{
-		Host:          "mail.example.com",
-		Port:          25,
-		From:          "[email protected]",
-		TemplatesPath: "templates",
-	}
-	err = smtpCfg.Initialize(configDir, true)
-	require.NoError(t, err)
-
-	err = check.Validate()
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "you must add a valid email address")
-
-	check.Email = "[email protected]"
-	err = check.Validate()
-	assert.NoError(t, err)
-
-	smtpCfg = smtp.Config{}
-	err = smtpCfg.Initialize(configDir, true)
-	require.NoError(t, err)
-
-	check.Notifications = []RetentionCheckNotification{RetentionCheckNotificationHook}
-	err = check.Validate()
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "data_retention_hook")
-
-	check.Notifications = []string{"not valid"}
-	err = check.Validate()
-	require.Error(t, err)
-	assert.Contains(t, err.Error(), "invalid notification")
-}
-
-func TestRetentionEmailNotifications(t *testing.T) {
-	smtpCfg := smtp.Config{
-		Host:          "127.0.0.1",
-		Port:          2525,
-		From:          "[email protected]",
-		TemplatesPath: "templates",
-	}
-	err := smtpCfg.Initialize(configDir, true)
-	require.NoError(t, err)
-
-	user := dataprovider.User{
-		BaseUser: sdk.BaseUser{
-			Username: "user1",
-		},
-	}
-	user.Permissions = make(map[string][]string)
-	user.Permissions["/"] = []string{dataprovider.PermAny}
-	check := RetentionCheck{
-		Notifications: []RetentionCheckNotification{RetentionCheckNotificationEmail},
-		Email:         "[email protected]",
-		results: []folderRetentionCheckResult{
-			{
-				Path:         "/",
-				Retention:    24,
-				DeletedFiles: 10,
-				DeletedSize:  32657,
-				Elapsed:      10 * time.Second,
-			},
-		},
-	}
-	conn := NewBaseConnection("", "", "", "", user)
-	conn.SetProtocol(ProtocolDataRetention)
-	conn.ID = fmt.Sprintf("data_retention_%v", user.Username)
-	check.conn = conn
-	check.sendNotifications(1*time.Second, nil)
-	err = check.sendEmailNotification(nil)
-	assert.NoError(t, err)
-	err = check.sendEmailNotification(errors.New("test error"))
-	assert.NoError(t, err)
-
-	check.results = nil
-	err = check.sendEmailNotification(nil)
-	if assert.Error(t, err) {
-		assert.Contains(t, err.Error(), "no data retention report available")
-	}
-
-	smtpCfg.Port = 2626
-	err = smtpCfg.Initialize(configDir, true)
-	require.NoError(t, err)
-	err = check.sendEmailNotification(nil)
-	assert.Error(t, err)
-	check.results = []folderRetentionCheckResult{
-		{
-			Path:         "/",
-			Retention:    24,
-			DeletedFiles: 20,
-			DeletedSize:  456789,
-			Elapsed:      12 * time.Second,
-		},
-	}
-
-	smtpCfg = smtp.Config{}
-	err = smtpCfg.Initialize(configDir, true)
-	require.NoError(t, err)
-	err = check.sendEmailNotification(nil)
-	assert.Error(t, err)
-}
-
-func TestRetentionHookNotifications(t *testing.T) {
-	dataRetentionHook := Config.DataRetentionHook
-
-	Config.DataRetentionHook = fmt.Sprintf("http://%v", httpAddr)
-	user := dataprovider.User{
-		BaseUser: sdk.BaseUser{
-			Username: "user2",
-		},
-	}
-	user.Permissions = make(map[string][]string)
-	user.Permissions["/"] = []string{dataprovider.PermAny}
-	check := RetentionCheck{
-		Notifications: []RetentionCheckNotification{RetentionCheckNotificationHook},
-		results: []folderRetentionCheckResult{
-			{
-				Path:         "/",
-				Retention:    24,
-				DeletedFiles: 10,
-				DeletedSize:  32657,
-				Elapsed:      10 * time.Second,
-			},
-		},
-	}
-	conn := NewBaseConnection("", "", "", "", user)
-	conn.SetProtocol(ProtocolDataRetention)
-	conn.ID = fmt.Sprintf("data_retention_%v", user.Username)
-	check.conn = conn
-	check.sendNotifications(1*time.Second, nil)
-	err := check.sendHookNotification(1*time.Second, nil)
-	assert.NoError(t, err)
-
-	Config.DataRetentionHook = fmt.Sprintf("http://%v/404", httpAddr)
-	err = check.sendHookNotification(1*time.Second, nil)
-	assert.ErrorIs(t, err, errUnexpectedHTTResponse)
-
-	Config.DataRetentionHook = "http://foo\x7f.com/retention"
-	err = check.sendHookNotification(1*time.Second, err)
-	assert.Error(t, err)
-
-	Config.DataRetentionHook = "relativepath"
-	err = check.sendHookNotification(1*time.Second, err)
-	assert.Error(t, err)
-
-	if runtime.GOOS != osWindows {
-		hookCmd, err := exec.LookPath("true")
-		assert.NoError(t, err)
-
-		Config.DataRetentionHook = hookCmd
-		err = check.sendHookNotification(1*time.Second, err)
-		assert.NoError(t, err)
-	}
-
-	Config.DataRetentionHook = dataRetentionHook
-}
-
 func TestRetentionPermissionsAndGetFolder(t *testing.T) {
 	user := dataprovider.User{
 		BaseUser: sdk.BaseUser{
@@ -311,7 +100,6 @@ func TestRetentionCheckAddRemove(t *testing.T) {
 				Retention: 48,
 			},
 		},
-		Notifications: []RetentionCheckNotification{RetentionCheckNotificationHook},
 	}
 	assert.NotNil(t, RetentionChecks.Add(check, &user))
 	checks := RetentionChecks.Get("")
@@ -321,8 +109,6 @@ func TestRetentionCheckAddRemove(t *testing.T) {
 	require.Len(t, checks[0].Folders, 1)
 	assert.Equal(t, check.Folders[0].Path, checks[0].Folders[0].Path)
 	assert.Equal(t, check.Folders[0].Retention, checks[0].Folders[0].Retention)
-	require.Len(t, checks[0].Notifications, 1)
-	assert.Equal(t, RetentionCheckNotificationHook, checks[0].Notifications[0])
 
 	assert.Nil(t, RetentionChecks.Add(check, &user))
 	assert.True(t, RetentionChecks.remove(username))
@@ -349,7 +135,6 @@ func TestRetentionCheckRole(t *testing.T) {
 				Retention: 48,
 			},
 		},
-		Notifications: []RetentionCheckNotification{RetentionCheckNotificationHook},
 	}
 	assert.NotNil(t, RetentionChecks.Add(check, &user))
 	checks := RetentionChecks.Get("")

internal/common/protocol_test.go

@@ -8343,7 +8343,12 @@ func TestRetentionAPI(t *testing.T) {
 				DeleteEmptyDirs: true,
 			},
 		}
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		check := common.RetentionCheck{
+			Folders: folderRetention,
+		}
+		c := common.RetentionChecks.Add(check, &user)
+		assert.NotNil(t, c)
+		err = c.Start()
 		assert.NoError(t, err)
 
 		assert.Eventually(t, func() bool {
@@ -8356,7 +8361,7 @@ func TestRetentionAPI(t *testing.T) {
 		err = client.Chtimes(uploadPath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
 		assert.NoError(t, err)
 
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		err = c.Start()
 		assert.NoError(t, err)
 
 		assert.Eventually(t, func() bool {
@@ -8374,11 +8379,13 @@ func TestRetentionAPI(t *testing.T) {
 		err = writeSFTPFile(uploadPath, 32, client)
 		assert.NoError(t, err)
 
-		folderRetention[0].DeleteEmptyDirs = false
+		check.Folders[0].DeleteEmptyDirs = false
 		err = client.Chtimes(uploadPath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
 		assert.NoError(t, err)
 
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		c = common.RetentionChecks.Add(check, &user)
+		assert.NotNil(t, c)
+		err = c.Start()
 		assert.NoError(t, err)
 
 		assert.Eventually(t, func() bool {
@@ -8432,7 +8439,12 @@ func TestRetentionAPI(t *testing.T) {
 				Retention: 0,
 			},
 		}
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		check := common.RetentionCheck{
+			Folders: folderRetention,
+		}
+		c := common.RetentionChecks.Add(check, &user)
+		assert.NotNil(t, c)
+		err = c.Start()
 		assert.NoError(t, err)
 
 		assert.Eventually(t, func() bool {
@@ -8453,7 +8465,12 @@ func TestRetentionAPI(t *testing.T) {
 			},
 		}
 
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		check = common.RetentionCheck{
+			Folders: folderRetention,
+		}
+		c = common.RetentionChecks.Add(check, &user)
+		assert.NotNil(t, c)
+		err = c.Start()
 		assert.NoError(t, err)
 
 		assert.Eventually(t, func() bool {
@@ -8487,8 +8504,13 @@ func TestRetentionAPI(t *testing.T) {
 			},
 		}
 
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
-		assert.NoError(t, err)
+		check := common.RetentionCheck{
+			Folders: folderRetention,
+		}
+		c := common.RetentionChecks.Add(check, &user)
+		assert.NotNil(t, c)
+		err = c.Start()
+		assert.ErrorIs(t, err, os.ErrPermission)
 
 		assert.Eventually(t, func() bool {
 			return len(common.RetentionChecks.Get("")) == 0
@@ -8497,8 +8519,10 @@ func TestRetentionAPI(t *testing.T) {
 		err = os.Chmod(dirPath, 0555)
 		assert.NoError(t, err)
 
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
-		assert.NoError(t, err)
+		c = common.RetentionChecks.Add(check, &user)
+		assert.NotNil(t, c)
+		err = c.Start()
+		assert.ErrorIs(t, err, os.ErrPermission)
 
 		assert.Eventually(t, func() bool {
 			return len(common.RetentionChecks.Get("")) == 0
@@ -8507,7 +8531,12 @@ func TestRetentionAPI(t *testing.T) {
 		err = os.Chmod(dirPath, os.ModePerm)
 		assert.NoError(t, err)
 
-		_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
+		check = common.RetentionCheck{
+			Folders: folderRetention,
+		}
+		c = common.RetentionChecks.Add(check, &user)
+		assert.NotNil(t, c)
+		err = c.Start()
 		assert.NoError(t, err)
 
 		assert.Eventually(t, func() bool {

internal/httpd/api_retention.go

@@ -15,13 +15,11 @@
 package httpd
 
 import (
-	"fmt"
 	"net/http"
 
 	"github.com/go-chi/render"
 
 	"github.com/drakkan/sftpgo/v2/internal/common"
-	"github.com/drakkan/sftpgo/v2/internal/dataprovider"
 )
 
 func getRetentionChecks(w http.ResponseWriter, r *http.Request) {
@@ -33,49 +31,3 @@ func getRetentionChecks(w http.ResponseWriter, r *http.Request) {
 	}
 	render.JSON(w, r, common.RetentionChecks.Get(claims.Role))
 }
-
-func startRetentionCheck(w http.ResponseWriter, r *http.Request) {
-	r.Body = http.MaxBytesReader(w, r.Body, maxRequestSize)
-	claims, err := getTokenClaims(r)
-	if err != nil || claims.Username == "" {
-		sendAPIResponse(w, r, err, "Invalid token claims", http.StatusBadRequest)
-		return
-	}
-	username := getURLParam(r, "username")
-	user, err := dataprovider.GetUserWithGroupSettings(username, claims.Role)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", getRespStatus(err))
-		return
-	}
-	var check common.RetentionCheck
-
-	err = render.DecodeJSON(r.Body, &check.Folders)
-	if err != nil {
-		sendAPIResponse(w, r, err, "", http.StatusBadRequest)
-		return
-	}
-
-	check.Notifications = getCommaSeparatedQueryParam(r, "notifications")
-	for _, notification := range check.Notifications {
-		if notification == common.RetentionCheckNotificationEmail {
-			admin, err := dataprovider.AdminExists(claims.Username)
-			if err != nil {
-				sendAPIResponse(w, r, err, "", getRespStatus(err))
-				return
-			}
-			check.Email = admin.Email
-		}
-	}
-	if err := check.Validate(); err != nil {
-		sendAPIResponse(w, r, err, "Invalid retention check", http.StatusBadRequest)
-		return
-	}
-	c := common.RetentionChecks.Add(check, &user)
-	if c == nil {
-		sendAPIResponse(w, r, err, fmt.Sprintf("Another check is already in progress for user %q", username),
-			http.StatusConflict)
-		return
-	}
-	go c.Start() //nolint:errcheck
-	sendAPIResponse(w, r, err, "Check started", http.StatusAccepted)
-}

internal/httpd/httpd_test.go

@@ -5109,6 +5109,13 @@ func TestRetentionAPI(t *testing.T) {
 	user, _, err := httpdtest.AddUser(getTestUser(), http.StatusCreated)
 	assert.NoError(t, err)
 
+	t.Cleanup(func() {
+		_, err = httpdtest.RemoveUser(user, http.StatusOK)
+		assert.NoError(t, err)
+		err = os.RemoveAll(user.GetHomeDir())
+		assert.NoError(t, err)
+	})
+
 	checks, _, err := httpdtest.GetRetentionChecks(http.StatusOK)
 	assert.NoError(t, err)
 	assert.Len(t, checks, 0)
@@ -5122,21 +5129,19 @@ func TestRetentionAPI(t *testing.T) {
 	folderRetention := []dataprovider.FolderRetention{
 		{
 			Path:            "/",
-			Retention:       0,
+			Retention:       24,
 			DeleteEmptyDirs: true,
 		},
 	}
 
-	_, err = httpdtest.StartRetentionCheck(altAdminUsername, folderRetention, http.StatusNotFound)
-	assert.NoError(t, err)
-
-	resp, err := httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusBadRequest)
-	assert.NoError(t, err)
-	assert.Contains(t, string(resp), "Invalid retention check")
+	check := common.RetentionCheck{
+		Folders: folderRetention,
+	}
+	c := common.RetentionChecks.Add(check, &user)
+	require.NotNil(t, c)
 
-	folderRetention[0].Retention = 24
-	_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
-	assert.NoError(t, err)
+	err = c.Start()
+	require.NoError(t, err)
 
 	assert.Eventually(t, func() bool {
 		return len(common.RetentionChecks.Get("")) == 0
@@ -5147,8 +5152,8 @@ func TestRetentionAPI(t *testing.T) {
 	err = os.Chtimes(localFilePath, time.Now().Add(-48*time.Hour), time.Now().Add(-48*time.Hour))
 	assert.NoError(t, err)
 
-	_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusAccepted)
-	assert.NoError(t, err)
+	err = c.Start()
+	require.NoError(t, err)
 
 	assert.Eventually(t, func() bool {
 		return len(common.RetentionChecks.Get("")) == 0
@@ -5157,54 +5162,25 @@ func TestRetentionAPI(t *testing.T) {
 	assert.NoFileExists(t, localFilePath)
 	assert.NoDirExists(t, filepath.Dir(localFilePath))
 
-	check := common.RetentionCheck{
-		Folders: folderRetention,
-	}
-	c := common.RetentionChecks.Add(check, &user)
+	c = common.RetentionChecks.Add(check, &user)
 	assert.NotNil(t, c)
 
-	_, err = httpdtest.StartRetentionCheck(user.Username, folderRetention, http.StatusConflict)
-	assert.NoError(t, err)
+	assert.Nil(t, common.RetentionChecks.Add(check, &user)) // a check for this user is already in progress
 
-	err = c.Start()
+	checks, _, err = httpdtest.GetRetentionChecks(http.StatusOK)
 	assert.NoError(t, err)
-	assert.Len(t, common.RetentionChecks.Get(""), 0)
+	assert.Len(t, checks, 1)
 
-	admin := getTestAdmin()
-	admin.Username = altAdminUsername
-	admin.Password = altAdminPassword
-	admin, _, err = httpdtest.AddAdmin(admin, http.StatusCreated)
-	assert.NoError(t, err)
-
-	token, err := getJWTAPITokenFromTestServer(altAdminUsername, altAdminPassword)
-	assert.NoError(t, err)
-	req, _ := http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check",
-		bytes.NewBuffer([]byte("invalid json")))
-	setBearerForReq(req, token)
-	rr := executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr)
-
-	asJSON, err := json.Marshal(folderRetention)
+	err = c.Start()
 	assert.NoError(t, err)
-	req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notifications=Email,",
-		bytes.NewBuffer(asJSON))
-	setBearerForReq(req, token)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusBadRequest, rr)
-	assert.Contains(t, rr.Body.String(), "to notify results via email")
 
-	_, err = httpdtest.RemoveAdmin(admin, http.StatusOK)
-	assert.NoError(t, err)
-	req, _ = http.NewRequest(http.MethodPost, retentionBasePath+"/"+user.Username+"/check?notifications=Email",
-		bytes.NewBuffer(asJSON))
-	setBearerForReq(req, token)
-	rr = executeRequest(req)
-	checkResponseCode(t, http.StatusNotFound, rr)
+	assert.Eventually(t, func() bool {
+		return len(common.RetentionChecks.Get("")) == 0
+	}, 1000*time.Millisecond, 50*time.Millisecond)
 
-	_, err = httpdtest.RemoveUser(user, http.StatusOK)
-	assert.NoError(t, err)
-	err = os.RemoveAll(user.GetHomeDir())
+	checks, _, err = httpdtest.GetRetentionChecks(http.StatusOK)
 	assert.NoError(t, err)
+	assert.Len(t, checks, 0)
 }
 
 func TestAddUserInvalidVirtualFolders(t *testing.T) {

internal/httpd/internal_test.go

@@ -1175,46 +1175,6 @@ func TestUpdateWebAdminInvalidClaims(t *testing.T) {
 	assert.Contains(t, rr.Body.String(), util.I18nErrorInvalidToken)
 }
 
-func TestRetentionInvalidTokenClaims(t *testing.T) {
-	username := "retentionuser"
-	user := dataprovider.User{
-		BaseUser: sdk.BaseUser{
-			Username: username,
-			Password: "pwd",
-			HomeDir:  filepath.Join(os.TempDir(), username),
-			Status:   1,
-		},
-	}
-	user.Permissions = make(map[string][]string)
-	user.Permissions["/"] = []string{dataprovider.PermAny}
-	user.Filters.AllowAPIKeyAuth = true
-	err := dataprovider.AddUser(&user, "", "", "")
-	assert.NoError(t, err)
-	folderRetention := []dataprovider.FolderRetention{
-		{
-			Path:            "/",
-			Retention:       0,
-			DeleteEmptyDirs: true,
-		},
-	}
-	asJSON, err := json.Marshal(folderRetention)
-	assert.NoError(t, err)
-	req, _ := http.NewRequest(http.MethodPost, retentionBasePath+"/"+username+"/check?notifications=Email", bytes.NewBuffer(asJSON))
-
-	rctx := chi.NewRouteContext()
-	rctx.URLParams.Add("username", username)
-
-	req = req.WithContext(context.WithValue(req.Context(), chi.RouteCtxKey, rctx))
-	req = req.WithContext(context.WithValue(req.Context(), jwtauth.ErrorCtxKey, errors.New("error")))
-	rr := httptest.NewRecorder()
-	startRetentionCheck(rr, req)
-	assert.Equal(t, http.StatusBadRequest, rr.Code)
-	assert.Contains(t, rr.Body.String(), "Invalid token claims")
-
-	err = dataprovider.DeleteUser(username, "", "", "")
-	assert.NoError(t, err)
-}
-
 func TestUpdateSMTPSecrets(t *testing.T) {
 	currentConfigs := &dataprovider.SMTPConfigs{
 		OAuth2: dataprovider.SMTPOAuth2{

internal/httpd/server.go

@@ -1430,8 +1430,6 @@ func (s *httpdServer) setupRESTAPIRoutes() {
 				router.With(s.checkPerms(dataprovider.PermAdminAny)).Delete(adminPath+"/{username}", deleteAdmin)
 				router.With(s.checkPerms(dataprovider.PermAdminDisableMFA)).Put(adminPath+"/{username}/2fa/disable", disableAdmin2FA)
 				router.With(s.checkPerms(dataprovider.PermAdminAny)).Get(retentionChecksPath, getRetentionChecks)
-				router.With(s.checkPerms(dataprovider.PermAdminAny)).Post(retentionBasePath+"/{username}/check",
-					startRetentionCheck)
 				router.With(s.checkPerms(dataprovider.PermAdminViewEvents), compressor.Handler).
 					Get(fsEventsPath, searchFsEvents)
 				router.With(s.checkPerms(dataprovider.PermAdminViewEvents), compressor.Handler).

internal/httpdtest/httpdtest.go

@@ -1173,20 +1173,6 @@ func GetRetentionChecks(expectedStatusCode int) ([]common.ActiveRetentionChecks,
 	return checks, body, err
 }
 
-// StartRetentionCheck starts a new retention check
-func StartRetentionCheck(username string, retention []dataprovider.FolderRetention, expectedStatusCode int) ([]byte, error) {
-	var body []byte
-	asJSON, _ := json.Marshal(retention)
-	resp, err := sendHTTPRequest(http.MethodPost, buildURLRelativeToBase(retentionBasePath, username, "check"),
-		bytes.NewBuffer(asJSON), "application/json", getDefaultToken())
-	if err != nil {
-		return body, err
-	}
-	defer resp.Body.Close()
-	body, _ = getResponseBody(resp)
-	return body, checkResponse(resp.StatusCode, expectedStatusCode)
-}
-
 // GetConnections returns status and stats for active SFTP/SCP connections
 func GetConnections(expectedStatusCode int) ([]common.ConnectionStatus, []byte, error) {
 	var connections []common.ConnectionStatus

openapi/openapi.yaml

@@ -1088,60 +1088,6 @@ paths:
           $ref: '#/components/responses/InternalServerError'
         default:
           $ref: '#/components/responses/DefaultResponse'
-  /retention/users/{username}/check:
-    parameters:
-      - name: username
-        in: path
-        description: the username
-        required: true
-        schema:
-          type: string
-      - name: notifications
-        in: query
-        description: 'specify how to notify results'
-        explode: false
-        schema:
-          type: array
-          items:
-            $ref: '#/components/schemas/RetentionCheckNotification'
-    post:
-      tags:
-        - data retention
-      summary: Start a retention check
-      description: 'Starts a new retention check for the given user. If a retention check for this user is already active a 409 status code is returned'
-      operationId: start_user_retention_check
-      requestBody:
-        required: true
-        description: 'Defines virtual paths to check and their retention time in hours'
-        content:
-          application/json:
-            schema:
-              type: array
-              items:
-                $ref: '#/components/schemas/FolderRetention'
-      responses:
-        '202':
-          description: successful operation
-          content:
-            application/json:
-              schema:
-                $ref: '#/components/schemas/ApiResponse'
-              example:
-                message: Check started
-        '400':
-          $ref: '#/components/responses/BadRequest'
-        '401':
-          $ref: '#/components/responses/Unauthorized'
-        '403':
-          $ref: '#/components/responses/Forbidden'
-        '404':
-          $ref: '#/components/responses/NotFound'
-        '409':
-          $ref: '#/components/responses/Conflict'
-        '500':
-          $ref: '#/components/responses/InternalServerError'
-        default:
-          $ref: '#/components/responses/DefaultResponse'
   /quotas/users/scans:
     get:
       tags:
@@ -5142,15 +5088,6 @@ components:
           * `shares-disabled` - sharing files and directories with external users is not allowed
           * `password-reset-disabled` - resetting the password is not allowed
           * `shares-without-password-disabled` - creating shares without password protection is not allowed
-    RetentionCheckNotification:
-      type: string
-      enum:
-        - Hook
-        - Email
-      description: |
-        Options:
-          * `Hook` - notify result using the defined hook. A "data_retention_hook" must be defined in your configuration file for this to work
-          * `Email` - notify results by email. The admin starting the retention check must have an associated email address and the SMTP server must be configured for this to work
     APIKeyScope:
       type: integer
       enum:
@@ -6271,9 +6208,6 @@ components:
         delete_empty_dirs:
           type: boolean
           description: if enabled, empty directories will be deleted
-        ignore_user_permissions:
-          type: boolean
-          description: 'if enabled, files will be deleted even if the user does not have the delete permission. The default is "false" which means that files will be skipped if the user does not have permission to delete them. File patterns filters will always be silently ignored'
     RetentionCheck:
       type: object
       properties:
@@ -6288,14 +6222,6 @@ components:
           type: integer
           format: int64
           description: check start time as unix timestamp in milliseconds
-        notifications:
-          type: array
-          items:
-            $ref: '#/components/schemas/RetentionCheckNotification'
-        email:
-          type: string
-          format: email
-          description: 'if the notification method is set to "Email", this is the e-mail address that receives the retention check report. This field is automatically set to the email address associated with the administrator starting the check'
     QuotaScan:
       type: object
       properties: