S3: add metrics

README.md

@@ -372,7 +372,7 @@ The program must write the questions on its standard output, in a single line, u
 - `instruction`, string. A short description to show to the user that is trying to authenticate. Can be empty or omitted
 - `questions`, list of questions to be asked to the user
 - `echos` list of boolean flags corresponding to the questions (so the lengths of both lists must be the same) and indicating whether user's reply for a particular question should be echoed on the screen while they are typing: true if it should be echoed, or false if it should be hidden.
-- `auth_result`, integer. Set this field to 1 to indicate successfull authentication, 0 is ignored, any other value means authentication error. If this fields is found and it is different from 0 then SFTPGo does not read any other questions from the external program and finalize the authentication.
+- `auth_result`, integer. Set this field to 1 to indicate successful authentication, 0 is ignored, any other value means authentication error. If this fields is found and it is different from 0 then SFTPGo does not read any other questions from the external program and finalize the authentication.
 
 SFTPGo writes the user answers to the program standard input, one per line, in the same order of the questions.
 Please be sure that your program receive the answers for all the issued questions before asking for the next ones.
@@ -411,7 +411,7 @@ Actions will not be executed if an error is detected and so a partial file is up
 
 The `command`, if defined, is invoked with the following arguments:
 
-- `action`, string, possibile values are: `download`, `upload`, `delete`, `rename`, `ssh_cmd`
+- `action`, string, possible values are: `download`, `upload`, `delete`, `rename`, `ssh_cmd`
 - `username`
 - `path` is the full filesystem path, can be empty for some ssh commands
 - `target_path`, non empty for `rename` action
@@ -446,7 +446,7 @@ Actions will not be fired for internal updates such as the last login or the use
 
 The `command`, if defined, is invoked with the following arguments:
 
-- `action`, string, possibile values are: `add`, `update`, `delete`
+- `action`, string, possible values are: `add`, `update`, `delete`
 - `username`
 - `ID`
 - `status`

go.mod

@@ -27,4 +27,4 @@ require (
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
 )
 
-replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172
+replace github.com/eikenb/pipeat v0.0.0-20190316224601-fb1f3a9aa29f => github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec

go.sum

@@ -32,8 +32,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
-github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172 h1:0mSDVf/0IPuuy3c5Qg+ceoxgbV7KUoLN3Ircswglf7A=
-github.com/drakkan/pipeat v0.0.0-20200122173221-ea03f92ba172/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w=
+github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec h1:DXfzg1NXoesnFzdCyyi2uU3o1o0XiWTN2ZcpWDE7MCk=
+github.com/drakkan/pipeat v0.0.0-20200123131427-11c048cfc0ec/go.mod h1:wNYvIpR5rIhoezOYcpxcXz4HbIEOu7A45EqlQCA+h+w=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=

metrics/metrics.go

@@ -19,40 +19,40 @@ var (
 		Help: "Total number of logged in users",
 	})
 
-	// totalUploads is the metric that reports the total number of uploads
+	// totalUploads is the metric that reports the total number of successful SFTP/SCP uploads
 	totalUploads = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_uploads_total",
-		Help: "The total number of uploads",
+		Help: "The total number of successful SFTP/SCP uploads",
 	})
 
-	// totalDownloads is the metric that reports the total number of downloads
+	// totalDownloads is the metric that reports the total number of successful SFTP/SCP downloads
 	totalDownloads = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_downloads_total",
-		Help: "The total number of downloads",
+		Help: "The total number of successful SFTP/SCP downloads",
 	})
 
-	// totalUploadErrors is the metric that reports the total number of upload errors
+	// totalUploadErrors is the metric that reports the total number of SFTP/SCP upload errors
 	totalUploadErrors = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_upload_errors_total",
-		Help: "The total number of upload errors",
+		Help: "The total number of SFTP/SCP upload errors",
 	})
 
-	// totalDownloadErrors is the metric that reports the total number of download errors
+	// totalDownloadErrors is the metric that reports the total number of SFTP/SCP download errors
 	totalDownloadErrors = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_download_errors_total",
-		Help: "The total number of download errors",
+		Help: "The total number of SFTP/SCP download errors",
 	})
 
-	// totalUploadSize is the metric that reports the total uploads size as bytes
+	// totalUploadSize is the metric that reports the total SFTP/SCP uploads size as bytes
 	totalUploadSize = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_upload_size",
-		Help: "The total upload size as bytes",
+		Help: "The total SFTP/SCP upload size as bytes, partial uploads are included",
 	})
 
-	// totalDownloadSize is the metric that reports the total downloads size as bytes
+	// totalDownloadSize is the metric that reports the total SFTP/SCP downloads size as bytes
 	totalDownloadSize = promauto.NewCounter(prometheus.CounterOpts{
 		Name: "sftpgo_download_size",
-		Help: "The total download size as bytes",
+		Help: "The total SFTP/SCP download size as bytes, partial downloads are included",
 	})
 
 	// totalSSHCommands is the metric that reports the total number of executed SSH commands
@@ -167,26 +167,188 @@ var (
 		Name: "sftpgo_http_server_errors_total",
 		Help: "The total number of HTTP requests served with 5xx status code",
 	})
+
+	// totalS3Uploads is the metric that reports the total number of successful S3 uploads
+	totalS3Uploads = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_uploads_total",
+		Help: "The total number of successful S3 uploads",
+	})
+
+	// totalS3Downloads is the metric that reports the total number of successful S3 downloads
+	totalS3Downloads = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_downloads_total",
+		Help: "The total number of successful S3 downloads",
+	})
+
+	// totalS3UploadErrors is the metric that reports the total number of S3 upload errors
+	totalS3UploadErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_upload_errors_total",
+		Help: "The total number of S3 upload errors",
+	})
+
+	// totalS3DownloadErrors is the metric that reports the total number of S3 download errors
+	totalS3DownloadErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_download_errors_total",
+		Help: "The total number of S3 download errors",
+	})
+
+	// totalS3UploadSize is the metric that reports the total S3 uploads size as bytes
+	totalS3UploadSize = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_upload_size",
+		Help: "The total S3 upload size as bytes, partial uploads are included",
+	})
+
+	// totalS3DownloadSize is the metric that reports the total S3 downloads size as bytes
+	totalS3DownloadSize = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_download_size",
+		Help: "The total S3 download size as bytes, partial downloads are included",
+	})
+
+	// totalS3ListObjects is the metric that reports the total successful S3 list objects requests
+	totalS3ListObjects = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_list_objects",
+		Help: "The total number of successful S3 list objects requests",
+	})
+
+	// totalS3CopyObject is the metric that reports the total successful S3 copy object requests
+	totalS3CopyObject = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_copy_object",
+		Help: "The total number of successful S3 copy object requests",
+	})
+
+	// totalS3DeleteObject is the metric that reports the total successful S3 delete object requests
+	totalS3DeleteObject = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_delete_object",
+		Help: "The total number of successful S3 delete object requests",
+	})
+
+	// totalS3ListObjectsError is the metric that reports the total S3 list objects errors
+	totalS3ListObjectsErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_list_objects_errors",
+		Help: "The total number of S3 list objects errors",
+	})
+
+	// totalS3CopyObjectErrors is the metric that reports the total S3 copy object errors
+	totalS3CopyObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_copy_object_errors",
+		Help: "The total number of S3 copy object errors",
+	})
+
+	// totalS3DeleteObjectErrors is the metric that reports the total S3 delete object errors
+	totalS3DeleteObjectErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_delete_object_errors",
+		Help: "The total number of S3 delete object errors",
+	})
+
+	// totalS3HeadBucket is the metric that reports the total successful S3 head bucket requests
+	totalS3HeadBucket = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_head_bucket",
+		Help: "The total number of successful S3 head bucket requests",
+	})
+
+	// totalS3CreateBucket is the metric that reports the total successful S3 create bucket requests
+	totalS3CreateBucket = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_create_bucket",
+		Help: "The total number of successful S3 create bucket requests",
+	})
+
+	// totalS3HeadBucketErrors is the metric that reports the total S3 head bucket errors
+	totalS3HeadBucketErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_head_bucket_errors",
+		Help: "The total number of S3 head bucket errors",
+	})
+
+	// totalS3CreateBucketErrors is the metric that reports the total S3 create bucket errors
+	totalS3CreateBucketErrors = promauto.NewCounter(prometheus.CounterOpts{
+		Name: "sftpgo_s3_create_bucket_errors",
+		Help: "The total number of S3 create bucket errors",
+	})
 )
 
-// TransferCompleted update metrics after an upload or a download
+// TransferCompleted updates metrics after an upload or a download
 func TransferCompleted(bytesSent, bytesReceived int64, transferKind int, err error) {
 	if transferKind == 0 {
 		// upload
 		if err == nil {
 			totalUploads.Inc()
-			totalUploadSize.Add(float64(bytesReceived))
 		} else {
 			totalUploadErrors.Inc()
 		}
+		totalUploadSize.Add(float64(bytesReceived))
 	} else {
 		// download
 		if err == nil {
 			totalDownloads.Inc()
-			totalDownloadSize.Add(float64(bytesSent))
 		} else {
 			totalDownloadErrors.Inc()
 		}
+		totalDownloadSize.Add(float64(bytesSent))
+	}
+}
+
+// S3TransferCompleted updates metrics after an S3 upload or a download
+func S3TransferCompleted(bytes int64, transferKind int, err error) {
+	if transferKind == 0 {
+		// upload
+		if err == nil {
+			totalS3Uploads.Inc()
+		} else {
+			totalS3UploadErrors.Inc()
+		}
+		totalS3UploadSize.Add(float64(bytes))
+	} else {
+		// download
+		if err == nil {
+			totalS3Downloads.Inc()
+		} else {
+			totalS3DownloadErrors.Inc()
+		}
+		totalS3DownloadSize.Add(float64(bytes))
+	}
+}
+
+// S3ListObjectsCompleted updates metrics after an S3 list objects request terminates
+func S3ListObjectsCompleted(err error) {
+	if err == nil {
+		totalS3ListObjects.Inc()
+	} else {
+		totalS3ListObjectsErrors.Inc()
+	}
+}
+
+// S3CopyObjectCompleted updates metrics after an S3 copy object request terminates
+func S3CopyObjectCompleted(err error) {
+	if err == nil {
+		totalS3CopyObject.Inc()
+	} else {
+		totalS3CopyObjectErrors.Inc()
+	}
+}
+
+// S3DeleteObjectCompleted updates metrics after an S3 delete object request terminates
+func S3DeleteObjectCompleted(err error) {
+	if err == nil {
+		totalS3DeleteObject.Inc()
+	} else {
+		totalS3DeleteObjectErrors.Inc()
+	}
+}
+
+// S3HeadBucketCompleted updates metrics after an S3 head bucket request terminates
+func S3HeadBucketCompleted(err error) {
+	if err == nil {
+		totalS3HeadBucket.Inc()
+	} else {
+		totalS3HeadBucketErrors.Inc()
+	}
+}
+
+// S3CreateBucketCompleted updates metrics after an S3 create bucket request terminates
+func S3CreateBucketCompleted(err error) {
+	if err == nil {
+		totalS3CreateBucket.Inc()
+	} else {
+		totalS3CreateBucketErrors.Inc()
 	}
 }
 

sftpd/transfer.go

@@ -132,6 +132,7 @@ func (t *Transfer) Close() error {
 		numFiles = 1
 	}
 	t.checkDownloadSize()
+	metrics.TransferCompleted(t.bytesSent, t.bytesReceived, t.transferType, t.transferError)
 	if t.transferType == transferUpload && t.file != nil && t.file.Name() != t.path {
 		if t.transferError == nil || uploadMode == uploadModeAtomicWithResume {
 			err = os.Rename(t.file.Name(), t.path)
@@ -162,7 +163,6 @@ func (t *Transfer) Close() error {
 			err = t.transferError
 		}
 	}
-	metrics.TransferCompleted(t.bytesSent, t.bytesReceived, t.transferType, t.transferError)
 	removeTransfer(t)
 	t.updateQuota(numFiles)
 	return err

vfs/s3fs.go

@@ -16,6 +16,7 @@ import (
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/drakkan/sftpgo/logger"
+	"github.com/drakkan/sftpgo/metrics"
 	"github.com/drakkan/sftpgo/utils"
 	"github.com/eikenb/pipeat"
 )
@@ -138,6 +139,7 @@ func (fs S3Fs) Stat(name string) (os.FileInfo, error) {
 		}
 		return true
 	})
+	metrics.S3ListObjectsCompleted(err)
 	if err == nil && len(result.Name()) == 0 {
 		err = errors.New("404 no such file or directory")
 	}
@@ -164,8 +166,9 @@ func (fs S3Fs) Open(name string) (*os.File, *pipeat.PipeReaderAt, func(), error)
 			Bucket: aws.String(fs.config.Bucket),
 			Key:    aws.String(key),
 		})
-		fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
 		w.CloseWithError(err)
+		fsLog(fs, logger.LevelDebug, "download completed, path: %#v size: %v, err: %v", name, n, err)
+		metrics.S3TransferCompleted(n, 1, err)
 	}()
 	return nil, r, cancelFn, nil
 }
@@ -187,8 +190,10 @@ func (fs S3Fs) Create(name string, flag int) (*os.File, *pipeat.PipeWriterAt, fu
 			Body:         r,
 			StorageClass: utils.NilIfEmpty(fs.config.StorageClass),
 		})
-		fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, err: %v", name, response, err)
 		r.CloseWithError(err)
+		fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, readed bytes: %v, err: %v",
+			name, response, r.GetReadedBytes(), err)
+		metrics.S3TransferCompleted(r.GetReadedBytes(), 0, err)
 	}()
 	return nil, w, cancelFn, nil
 }
@@ -229,6 +234,7 @@ func (fs S3Fs) Rename(source, target string) error {
 		CopySource: aws.String(copySource),
 		Key:        aws.String(target),
 	})
+	metrics.S3CopyObjectCompleted(err)
 	if err != nil {
 		return err
 	}
@@ -255,6 +261,7 @@ func (fs S3Fs) Remove(name string, isDir bool) error {
 		Bucket: aws.String(fs.config.Bucket),
 		Key:    aws.String(name),
 	})
+	metrics.S3DeleteObjectCompleted(err)
 	return err
 }
 
@@ -331,6 +338,7 @@ func (fs S3Fs) ReadDir(dirname string) ([]os.FileInfo, error) {
 		}
 		return true
 	})
+	metrics.S3ListObjectsCompleted(err)
 	return result, err
 }
 
@@ -401,6 +409,7 @@ func (fs S3Fs) CheckRootPath(username string, uid int, gid int) bool {
 	_, err = fs.svc.CreateBucketWithContext(ctx, input)
 	fsLog(fs, logger.LevelDebug, "bucket %#v for user %#v does not exists, try to create, error: %v",
 		fs.config.Bucket, username, err)
+	metrics.S3CreateBucketCompleted(err)
 	return err == nil
 }
 
@@ -421,7 +430,7 @@ func (fs S3Fs) ScanRootDirContents() (int, int64, error) {
 		}
 		return true
 	})
-
+	metrics.S3ListObjectsCompleted(err)
 	return numFiles, size, err
 }
 
@@ -496,6 +505,7 @@ func (fs *S3Fs) checkIfBucketExists() error {
 	_, err := fs.svc.HeadBucketWithContext(ctx, &s3.HeadBucketInput{
 		Bucket: aws.String(fs.config.Bucket),
 	})
+	metrics.S3HeadBucketCompleted(err)
 	return err
 }