API data retention check: send CSV reports for email notifications

replace the HTML email with the same CSV report used in the
event manager

Signed-off-by: Nicola Murino <[email protected]>

go.mod

@@ -60,7 +60,7 @@ require (
 	github.com/studio-b12/gowebdav v0.0.0-20220128162035-c7b1ff8a5e62
 	github.com/unrolled/secure v1.13.0
 	github.com/wagslane/go-password-validator v0.3.0
-	github.com/xhit/go-simple-mail/v2 v2.11.0
+	github.com/xhit/go-simple-mail/v2 v2.12.0
 	github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a
 	go.etcd.io/bbolt v1.3.6
 	go.uber.org/automaxprocs v1.5.1
@@ -68,16 +68,16 @@ require (
 	golang.org/x/crypto v0.0.0-20220924013350-4ba4fb4dd9e7
 	golang.org/x/net v0.0.0-20220923203811-8be639271d50
 	golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1
-	golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25
+	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec
 	golang.org/x/time v0.0.0-20220922220347-f3bd1da661af
-	google.golang.org/api v0.97.0
+	google.golang.org/api v0.98.0
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
 require (
 	cloud.google.com/go v0.104.0 // indirect
 	cloud.google.com/go/compute v1.10.0 // indirect
-	cloud.google.com/go/iam v0.4.0 // indirect
+	cloud.google.com/go/iam v0.5.0 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.1 // indirect
 	github.com/ajg/form v1.5.1 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect
@@ -156,7 +156,7 @@ require (
 	golang.org/x/tools v0.1.12 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220923205249-dd2d53f1fffc // indirect
+	google.golang.org/genproto v0.0.0-20220927151529-dcaddaf36704 // indirect
 	google.golang.org/grpc v1.49.0 // indirect
 	google.golang.org/protobuf v1.28.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect

go.sum

@@ -55,8 +55,8 @@ cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx
 cloud.google.com/go/iam v0.1.0/go.mod h1:vcUNEa0pEm0qRVpmWepWaFMIAI8/hjB9mO8rNCJtF6c=
 cloud.google.com/go/iam v0.1.1/go.mod h1:CKqrcnI/suGpybEHxZ7BMehL0oA4LpdyJdUlTl9jVMw=
 cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY=
-cloud.google.com/go/iam v0.4.0 h1:YBYU00SCDzZJdHqVc4I5d6lsklcYIjQZa1YmEz4jlSE=
-cloud.google.com/go/iam v0.4.0/go.mod h1:cbaZxyScUhxl7ZAkNWiALgihfP75wS/fUsVNaa1r3vA=
+cloud.google.com/go/iam v0.5.0 h1:fz9X5zyTWBmamZsqvqZqD7khbifcZF/q+Z1J8pfhIUg=
+cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc=
 cloud.google.com/go/kms v1.1.0/go.mod h1:WdbppnCDMDpOvoYBMn1+gNmOeEoZYqAv+HeuKARGCXI=
 cloud.google.com/go/kms v1.4.0 h1:iElbfoE61VeLhnZcGOltqL8HIly8Nhbe5t6JlH9GXjo=
 cloud.google.com/go/kms v1.4.0/go.mod h1:fajBHndQ+6ubNw6Ss2sSd+SWvjL26RNo/dr7uxsnnOA=
@@ -774,8 +774,8 @@ github.com/unrolled/secure v1.13.0 h1:sdr3Phw2+f8Px8HE5sd1EHdj1aV3yUwed/uZXChLFs
 github.com/unrolled/secure v1.13.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
 github.com/wagslane/go-password-validator v0.3.0 h1:vfxOPzGHkz5S146HDpavl0cw1DSVP061Ry2PX0/ON6I=
 github.com/wagslane/go-password-validator v0.3.0/go.mod h1:TI1XJ6T5fRdRnHqHt14pvy1tNVnrwe7m3/f1f2fDphQ=
-github.com/xhit/go-simple-mail/v2 v2.11.0 h1:o/056V50zfkO3Mm5tVdo9rG3ryg4ZmJ2XW5GMinHfVs=
-github.com/xhit/go-simple-mail/v2 v2.11.0/go.mod h1:b7P5ygho6SYE+VIqpxA6QkYfv4teeyG4MKqB3utRu98=
+github.com/xhit/go-simple-mail/v2 v2.12.0 h1:KweA6NO8Z6fZyeckMPNpvElU6QDIyBShlpce1sYUZgg=
+github.com/xhit/go-simple-mail/v2 v2.12.0/go.mod h1:b7P5ygho6SYE+VIqpxA6QkYfv4teeyG4MKqB3utRu98=
 github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a h1:XfF01GyP+0eWCaVp0y6rNN+kFp7pt9Da4UUYrJ5XPWA=
 github.com/yl2chen/cidranger v1.0.3-0.20210928021809-d1cb2c52f37a/go.mod h1:aXb8yZQEWo1XHGMf1qQfnb83GR/EJ2EBlwtUgAaNBoE=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -981,8 +981,8 @@ golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25 h1:nwzwVf0l2Y/lkov/+IYgMMbFyI+QypZDds9RxlSmsFQ=
-golang.org/x/sys v0.0.0-20220926163933-8cfa568d3c25/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=
+golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -1124,8 +1124,8 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69
 google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
 google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
 google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
-google.golang.org/api v0.97.0 h1:x/vEL1XDF/2V4xzdNgFPaKHluRESo2aTsL7QzHnBtGQ=
-google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
+google.golang.org/api v0.98.0 h1:yxZrcxXESimy6r6mdL5Q6EnZwmewDJK2dVg3g75s5Dg=
+google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1230,8 +1230,8 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP
 google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
 google.golang.org/genproto v0.0.0-20220624142145-8cd45d7dbd1f/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA=
-google.golang.org/genproto v0.0.0-20220923205249-dd2d53f1fffc h1:saaNe2+SBQxandnzcD/qB1JEBQ2Pqew+KlFLLdA/XcM=
-google.golang.org/genproto v0.0.0-20220923205249-dd2d53f1fffc/go.mod h1:yEEpwVWKMZZzo81NwRgyEJnA2fQvpXAYPVisv8EgDVs=
+google.golang.org/genproto v0.0.0-20220927151529-dcaddaf36704 h1:H1AcWFV69NFCMeBJ8nVLtv8uHZZ5Ozcgoq012hHEFuU=
+google.golang.org/genproto v0.0.0-20220927151529-dcaddaf36704/go.mod h1:woMGP53BroOrRY3xTxlbr8Y3eB/nzAvvFM83q7kG2OI=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=

internal/common/dataretention.go

@@ -29,6 +29,8 @@ import (
 	"sync"
 	"time"
 
+	mail "github.com/xhit/go-simple-mail/v2"
+
 	"github.com/drakkan/sftpgo/v2/internal/command"
 	"github.com/drakkan/sftpgo/v2/internal/dataprovider"
 	"github.com/drakkan/sftpgo/v2/internal/httpclient"
@@ -350,41 +352,41 @@ func (c *RetentionCheck) sendNotifications(elapsed time.Duration, err error) {
 	for _, notification := range c.Notifications {
 		switch notification {
 		case RetentionCheckNotificationEmail:
-			c.sendEmailNotification(elapsed, err) //nolint:errcheck
+			c.sendEmailNotification(err) //nolint:errcheck
 		case RetentionCheckNotificationHook:
 			c.sendHookNotification(elapsed, err) //nolint:errcheck
 		}
 	}
 }
 
-func (c *RetentionCheck) sendEmailNotification(elapsed time.Duration, errCheck error) error {
-	body := new(bytes.Buffer)
-	data := make(map[string]any)
-	data["Results"] = c.results
-	totalDeletedFiles := 0
-	totalDeletedSize := int64(0)
-	for _, result := range c.results {
-		totalDeletedFiles += result.DeletedFiles
-		totalDeletedSize += result.DeletedSize
-	}
-	data["HumanizeSize"] = util.ByteCountIEC
-	data["TotalFiles"] = totalDeletedFiles
-	data["TotalSize"] = totalDeletedSize
-	data["Elapsed"] = elapsed
-	data["Username"] = c.conn.User.Username
-	data["StartTime"] = util.GetTimeFromMsecSinceEpoch(c.StartTime)
-	if errCheck == nil {
-		data["Status"] = "Succeeded"
-	} else {
-		data["Status"] = "Failed"
+func (c *RetentionCheck) sendEmailNotification(errCheck error) error {
+	params := EventParams{}
+	if len(c.results) > 0 || errCheck != nil {
+		params.retentionChecks = append(params.retentionChecks, executedRetentionCheck{
+			Username:   c.conn.User.Username,
+			ActionName: "Retention check",
+			Results:    c.results,
+		})
 	}
-	if err := smtp.RenderRetentionReportTemplate(body, data); err != nil {
-		c.conn.Log(logger.LevelError, "unable to render retention check template: %v", err)
+	var files []mail.File
+	f, err := params.getRetentionReportsAsMailAttachment()
+	if err != nil {
+		c.conn.Log(logger.LevelError, "unable to get retention report as mail attachment: %v", err)
 		return err
 	}
+	f.Name = "retention-report.zip"
+	files = append(files, f)
+
 	startTime := time.Now()
-	subject := fmt.Sprintf("Retention check completed for user %#v", c.conn.User.Username)
-	if err := smtp.SendEmail([]string{c.Email}, subject, body.String(), smtp.EmailContentTypeTextHTML); err != nil {
+	var subject string
+	if errCheck == nil {
+		subject = fmt.Sprintf("Successful retention check for user %q", c.conn.User.Username)
+	} else {
+		subject = fmt.Sprintf("Retention check failed for user %q", c.conn.User.Username)
+	}
+	body := "Further details attached."
+	err = smtp.SendEmail([]string{c.Email}, subject, body, smtp.EmailContentTypeTextPlain, files...)
+	if err != nil {
 		c.conn.Log(logger.LevelError, "unable to notify retention check result via email: %v, elapsed: %v", err,
 			time.Since(startTime))
 		return err

internal/common/dataretention_test.go

@@ -146,21 +146,36 @@ func TestRetentionEmailNotifications(t *testing.T) {
 	conn.ID = fmt.Sprintf("data_retention_%v", user.Username)
 	check.conn = conn
 	check.sendNotifications(1*time.Second, nil)
-	err = check.sendEmailNotification(1*time.Second, nil)
+	err = check.sendEmailNotification(nil)
 	assert.NoError(t, err)
-	err = check.sendEmailNotification(1*time.Second, errors.New("test error"))
+	err = check.sendEmailNotification(errors.New("test error"))
 	assert.NoError(t, err)
 
+	check.results = nil
+	err = check.sendEmailNotification(nil)
+	if assert.Error(t, err) {
+		assert.Contains(t, err.Error(), "no data retention report available")
+	}
+
 	smtpCfg.Port = 2626
 	err = smtpCfg.Initialize(configDir)
 	require.NoError(t, err)
-	err = check.sendEmailNotification(1*time.Second, nil)
+	err = check.sendEmailNotification(nil)
 	assert.Error(t, err)
+	check.results = []folderRetentionCheckResult{
+		{
+			Path:         "/",
+			Retention:    24,
+			DeletedFiles: 20,
+			DeletedSize:  456789,
+			Elapsed:      12 * time.Second,
+		},
+	}
 
 	smtpCfg = smtp.Config{}
 	err = smtpCfg.Initialize(configDir)
 	require.NoError(t, err)
-	err = check.sendEmailNotification(1*time.Second, nil)
+	err = check.sendEmailNotification(nil)
 	assert.Error(t, err)
 }
 

internal/smtp/smtp.go

@@ -43,9 +43,8 @@ const (
 )
 
 const (
-	templateEmailDir             = "email"
-	templateRetentionCheckResult = "retention-check-report.html"
-	templatePasswordReset        = "reset-password.html"
+	templateEmailDir      = "email"
+	templatePasswordReset = "reset-password.html"
 )
 
 var (
@@ -156,24 +155,13 @@ func (c *Config) getAuthType() mail.AuthType {
 
 func loadTemplates(templatesPath string) {
 	logger.Debug(logSender, "", "loading templates from %#v", templatesPath)
-	retentionCheckPath := filepath.Join(templatesPath, templateRetentionCheckResult)
-	retentionTmpl := util.LoadTemplate(nil, retentionCheckPath)
 
 	passwordResetPath := filepath.Join(templatesPath, templatePasswordReset)
 	pwdResetTmpl := util.LoadTemplate(nil, passwordResetPath)
 
-	emailTemplates[templateRetentionCheckResult] = retentionTmpl
 	emailTemplates[templatePasswordReset] = pwdResetTmpl
 }
 
-// RenderRetentionReportTemplate executes the retention report template
-func RenderRetentionReportTemplate(buf *bytes.Buffer, data any) error {
-	if smtpServer == nil {
-		return errors.New("smtp: not configured")
-	}
-	return emailTemplates[templateRetentionCheckResult].Execute(buf, data)
-}
-
 // RenderPasswordResetTemplate executes the password reset template
 func RenderPasswordResetTemplate(buf *bytes.Buffer, data any) error {
 	if smtpServer == nil {

pkgs/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-NFPM_VERSION=2.18.1
+NFPM_VERSION=2.19.1
 NFPM_ARCH=${NFPM_ARCH:-amd64}
 if [ -z ${SFTPGO_VERSION} ]
 then

templates/email/retention-check-report.html

@@ -1,46 +0,0 @@
-<!--
-Copyright (C) 2019-2022  Nicola Murino
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU Affero General Public License as published
-by the Free Software Foundation, version 3.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-GNU Affero General Public License for more details.
-
-You should have received a copy of the GNU Affero General Public License
-along with this program.  If not, see <https://www.gnu.org/licenses/>.
--->
-Retention check report for user <b>"{{.Username}}"</b>
-<br><br>
-Status: <strong>{{.Status}}</strong>
-<br>
-Start time: {{.StartTime}}
-<br>
-Total files deleted: {{.TotalFiles}}
-<br>
-Total size deleted: {{call .HumanizeSize .TotalSize}}
-<br>
-Elapsed: {{.Elapsed}}
-<br>
-{{range .Results -}}
-<p>
-    Path: {{.Path}}
-    {{- if .Error}}
-    <br>
-    Error: {{.Error}}
-    {{- end}}
-    {{- if .Info}}
-    <br>
-    Info: {{.Info}}
-    {{- end}}
-    <br>
-    Retention: {{.Retention}} hours
-    <br>
-    Files deleted: {{.DeletedFiles}}
-    <br>
-    Size deleted: {{call $.HumanizeSize .DeletedSize}}
-</p>
-{{end}}