S3: add ACL support

Fixes #610

cmd/portable.go

@@ -44,6 +44,7 @@ var (
 	portableS3AccessSecret             string
 	portableS3Endpoint                 string
 	portableS3StorageClass             string
+	portableS3ACL                      string
 	portableS3KeyPrefix                string
 	portableS3ULPartSize               int
 	portableS3ULConcurrency            int
@@ -169,6 +170,7 @@ Please take a look at the usage below to customize the serving parameters`,
 								AccessSecret:      kms.NewPlainSecret(portableS3AccessSecret),
 								Endpoint:          portableS3Endpoint,
 								StorageClass:      portableS3StorageClass,
+								ACL:               portableS3ACL,
 								KeyPrefix:         portableS3KeyPrefix,
 								UploadPartSize:    int64(portableS3ULPartSize),
 								UploadConcurrency: portableS3ULConcurrency,
@@ -288,6 +290,7 @@ sftpfs => SFTP (legacy: 5)`)
 	portableCmd.Flags().StringVar(&portableS3AccessSecret, "s3-access-secret", "", "")
 	portableCmd.Flags().StringVar(&portableS3Endpoint, "s3-endpoint", "", "")
 	portableCmd.Flags().StringVar(&portableS3StorageClass, "s3-storage-class", "", "")
+	portableCmd.Flags().StringVar(&portableS3ACL, "s3-acl", "", "")
 	portableCmd.Flags().StringVar(&portableS3KeyPrefix, "s3-key-prefix", "", `Allows to restrict access to the
 virtual folder identified by this
 prefix and its contents`)

docs/portable-mode.md

@@ -81,6 +81,7 @@ Flags:
   -k, --public-key strings
       --s3-access-key string
       --s3-access-secret string
+      --s3-acl string
       --s3-bucket string
       --s3-endpoint string
       --s3-key-prefix string            Allows to restrict access to the

httpd/httpd_test.go

@@ -1543,6 +1543,7 @@ func TestUserRedactedPassword(t *testing.T) {
 	u.FsConfig.S3Config.AccessSecret = kms.NewSecret(kms.SecretStatusRedacted, "access-secret", "", "")
 	u.FsConfig.S3Config.Endpoint = "http://127.0.0.1:9000/path?k=m"
 	u.FsConfig.S3Config.StorageClass = "Standard"
+	u.FsConfig.S3Config.ACL = "bucket-owner-full-control"
 	_, resp, err := httpdtest.AddUser(u, http.StatusBadRequest)
 	assert.NoError(t, err, string(resp))
 	assert.Contains(t, string(resp), "cannot save a user with a redacted secret")
@@ -13181,6 +13182,7 @@ func TestWebUserS3Mock(t *testing.T) {
 	user.FsConfig.S3Config.DownloadPartSize = 6
 	user.FsConfig.S3Config.DownloadConcurrency = 3
 	user.FsConfig.S3Config.ForcePathStyle = true
+	user.FsConfig.S3Config.ACL = "public-read"
 	user.Description = "s3 tèst user"
 	form := make(url.Values)
 	form.Set(csrfFormToken, csrfToken)
@@ -13205,6 +13207,7 @@ func TestWebUserS3Mock(t *testing.T) {
 	form.Set("s3_access_key", user.FsConfig.S3Config.AccessKey)
 	form.Set("s3_access_secret", user.FsConfig.S3Config.AccessSecret.GetPayload())
 	form.Set("s3_storage_class", user.FsConfig.S3Config.StorageClass)
+	form.Set("s3_acl", user.FsConfig.S3Config.ACL)
 	form.Set("s3_endpoint", user.FsConfig.S3Config.Endpoint)
 	form.Set("s3_key_prefix", user.FsConfig.S3Config.KeyPrefix)
 	form.Set("pattern_path0", "/dir1")
@@ -13282,6 +13285,7 @@ func TestWebUserS3Mock(t *testing.T) {
 	assert.Equal(t, updateUser.FsConfig.S3Config.Region, user.FsConfig.S3Config.Region)
 	assert.Equal(t, updateUser.FsConfig.S3Config.AccessKey, user.FsConfig.S3Config.AccessKey)
 	assert.Equal(t, updateUser.FsConfig.S3Config.StorageClass, user.FsConfig.S3Config.StorageClass)
+	assert.Equal(t, updateUser.FsConfig.S3Config.ACL, user.FsConfig.S3Config.ACL)
 	assert.Equal(t, updateUser.FsConfig.S3Config.Endpoint, user.FsConfig.S3Config.Endpoint)
 	assert.Equal(t, updateUser.FsConfig.S3Config.KeyPrefix, user.FsConfig.S3Config.KeyPrefix)
 	assert.Equal(t, updateUser.FsConfig.S3Config.UploadPartSize, user.FsConfig.S3Config.UploadPartSize)
@@ -13931,6 +13935,7 @@ func TestS3WebFolderMock(t *testing.T) {
 	S3AccessSecret := kms.NewPlainSecret("folder-access-secret")
 	S3Endpoint := "http://127.0.0.1:9000/path?b=c"
 	S3StorageClass := "Standard"
+	S3ACL := "public-read-write"
 	S3KeyPrefix := "somedir/subdir/"
 	S3UploadPartSize := 5
 	S3UploadConcurrency := 4
@@ -13947,6 +13952,7 @@ func TestS3WebFolderMock(t *testing.T) {
 	form.Set("s3_access_key", S3AccessKey)
 	form.Set("s3_access_secret", S3AccessSecret.GetPayload())
 	form.Set("s3_storage_class", S3StorageClass)
+	form.Set("s3_acl", S3ACL)
 	form.Set("s3_endpoint", S3Endpoint)
 	form.Set("s3_key_prefix", S3KeyPrefix)
 	form.Set("s3_upload_part_size", strconv.Itoa(S3UploadPartSize))
@@ -13991,6 +13997,7 @@ func TestS3WebFolderMock(t *testing.T) {
 	assert.NotEmpty(t, folder.FsConfig.S3Config.AccessSecret.GetPayload())
 	assert.Equal(t, S3Endpoint, folder.FsConfig.S3Config.Endpoint)
 	assert.Equal(t, S3StorageClass, folder.FsConfig.S3Config.StorageClass)
+	assert.Equal(t, S3ACL, folder.FsConfig.S3Config.ACL)
 	assert.Equal(t, S3KeyPrefix, folder.FsConfig.S3Config.KeyPrefix)
 	assert.Equal(t, S3UploadConcurrency, folder.FsConfig.S3Config.UploadConcurrency)
 	assert.Equal(t, int64(S3UploadPartSize), folder.FsConfig.S3Config.UploadPartSize)

httpd/schema/openapi.yaml

@@ -4181,6 +4181,9 @@ components:
           description: optional endpoint
         storage_class:
           type: string
+        acl:
+          type: string
+          description: 'The canned ACL to apply to uploaded objects. Leave empty to use the default ACL. For more information and available ACLs, see here: https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl'
         upload_part_size:
           type: integer
           description: 'the buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB, and if this value is set to zero, the default value (5MB) for the AWS SDK will be used. The minimum allowed value is 5.'

httpd/webadmin.go

@@ -830,6 +830,7 @@ func getS3Config(r *http.Request) (vfs.S3FsConfig, error) {
 	config.AccessSecret = getSecretFromFormField(r, "s3_access_secret")
 	config.Endpoint = r.Form.Get("s3_endpoint")
 	config.StorageClass = r.Form.Get("s3_storage_class")
+	config.ACL = r.Form.Get("s3_acl")
 	config.KeyPrefix = r.Form.Get("s3_key_prefix")
 	config.UploadPartSize, err = strconv.ParseInt(r.Form.Get("s3_upload_part_size"), 10, 64)
 	if err != nil {

httpdtest/httpdtest.go

@@ -1245,7 +1245,7 @@ func compareFsConfig(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
 	return compareSFTPFsConfig(expected, actual)
 }
 
-func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
+func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error { //nolint:gocyclo
 	if expected.S3Config.Bucket != actual.S3Config.Bucket {
 		return errors.New("fs S3 bucket mismatch")
 	}
@@ -1264,6 +1264,9 @@ func compareS3Config(expected *vfs.Filesystem, actual *vfs.Filesystem) error {
 	if expected.S3Config.StorageClass != actual.S3Config.StorageClass {
 		return errors.New("fs S3 storage class mismatch")
 	}
+	if expected.S3Config.ACL != actual.S3Config.ACL {
+		return errors.New("fs S3 ACL mismatch")
+	}
 	if expected.S3Config.UploadPartSize != actual.S3Config.UploadPartSize {
 		return errors.New("fs S3 upload part size mismatch")
 	}

sdk/filesystem.go

@@ -99,6 +99,10 @@ type S3FsConfig struct {
 	AccessSecret *kms.Secret `json:"access_secret,omitempty"`
 	Endpoint     string      `json:"endpoint,omitempty"`
 	StorageClass string      `json:"storage_class,omitempty"`
+	// The canned ACL to apply to uploaded objects. Leave empty to use the default ACL.
+	// For more information and available ACLs, see here:
+	// https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl
+	ACL string `json:"acl,omitempty"`
 	// The buffer size (in MB) to use for multipart uploads. The minimum allowed part size is 5MB,
 	// and if this value is set to zero, the default value (5MB) for the AWS SDK will be used.
 	// The minimum allowed value is 5.

templates/webadmin/fsconfig.html

@@ -109,8 +109,19 @@
                 </small>
             </div>
             <div class="col-sm-2"></div>
-            <label for="idS3KeyPrefix" class="col-sm-2 col-form-label">Key Prefix</label>
+            <label for="idS3KeyPrefix" class="col-sm-2 col-form-label">ACL</label>
             <div class="col-sm-3">
+                <input type="text" class="form-control" id="idS3ACL" name="s3_acl" placeholder=""
+                    value="{{.S3Config.ACL}}" maxlength="255" aria-describedby="S3ACLHelpBlock">
+                <small id="S3ACLHelpBlock" class="form-text text-muted">
+                    ACL for uploaded objects. For more info take a look <a href="https://docs.aws.amazon.com/AmazonS3/latest/userguide/acl-overview.html#canned-acl" target="_blank">here</a>
+                </small>
+            </div>
+        </div>
+
+        <div class="form-group row fsconfig fsconfig-s3fs">
+            <label for="idS3KeyPrefix" class="col-sm-2 col-form-label">Key Prefix</label>
+            <div class="col-sm-10">
                 <input type="text" class="form-control" id="idS3KeyPrefix" name="s3_key_prefix" placeholder=""
                     value="{{.S3Config.KeyPrefix}}" maxlength="255" aria-describedby="S3KeyPrefixHelpBlock">
                 <small id="S3KeyPrefixHelpBlock" class="form-text text-muted">
@@ -119,6 +130,7 @@
             </div>
         </div>
 
+
         <div class="form-group fsconfig fsconfig-s3fs">
             <div class="form-check">
                 <input type="checkbox" class="form-check-input" id="idS3ForcePathStyle" name="s3_force_path_style"

vfs/filesystem.go

@@ -235,6 +235,7 @@ func (f *Filesystem) GetACopy() Filesystem {
 				AccessSecret:        f.S3Config.AccessSecret.Clone(),
 				Endpoint:            f.S3Config.Endpoint,
 				StorageClass:        f.S3Config.StorageClass,
+				ACL:                 f.S3Config.ACL,
 				KeyPrefix:           f.S3Config.KeyPrefix,
 				UploadPartSize:      f.S3Config.UploadPartSize,
 				UploadConcurrency:   f.S3Config.UploadConcurrency,

vfs/s3fs.go

@@ -244,6 +244,7 @@ func (fs *S3Fs) Create(name string, flag int) (File, *PipeWriter, func(), error)
 			Bucket:       aws.String(fs.config.Bucket),
 			Key:          aws.String(key),
 			Body:         r,
+			ACL:          util.NilIfEmpty(fs.config.ACL),
 			StorageClass: util.NilIfEmpty(fs.config.StorageClass),
 			ContentType:  util.NilIfEmpty(contentType),
 		}, func(u *s3manager.Uploader) {
@@ -252,8 +253,8 @@ func (fs *S3Fs) Create(name string, flag int) (File, *PipeWriter, func(), error)
 		})
 		r.CloseWithError(err) //nolint:errcheck
 		p.Done(err)
-		fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, response: %v, readed bytes: %v, err: %+v",
-			name, response, r.GetReadedBytes(), err)
+		fsLog(fs, logger.LevelDebug, "upload completed, path: %#v, acl: %#v, response: %v, readed bytes: %v, err: %+v",
+			name, fs.config.ACL, response, r.GetReadedBytes(), err)
 		metric.S3TransferCompleted(r.GetReadedBytes(), 0, err)
 	}()
 	return nil, p, cancelFn, nil
@@ -306,6 +307,7 @@ func (fs *S3Fs) Rename(source, target string) error {
 		CopySource:   aws.String(pathEscape(copySource)),
 		Key:          aws.String(target),
 		StorageClass: util.NilIfEmpty(fs.config.StorageClass),
+		ACL:          util.NilIfEmpty(fs.config.ACL),
 		ContentType:  util.NilIfEmpty(contentType),
 	})
 	if err != nil {

vfs/vfs.go

@@ -169,6 +169,9 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool {
 	if c.StorageClass != other.StorageClass {
 		return false
 	}
+	if c.ACL != other.ACL {
+		return false
+	}
 	if c.UploadPartSize != other.UploadPartSize {
 		return false
 	}
@@ -187,6 +190,10 @@ func (c *S3FsConfig) isEqual(other *S3FsConfig) bool {
 	if c.ForcePathStyle != other.ForcePathStyle {
 		return false
 	}
+	return c.isSecretEqual(other)
+}
+
+func (c *S3FsConfig) isSecretEqual(other *S3FsConfig) bool {
 	if c.AccessSecret == nil {
 		c.AccessSecret = kms.NewEmptySecret()
 	}
@@ -263,6 +270,8 @@ func (c *S3FsConfig) Validate() error {
 			c.KeyPrefix += "/"
 		}
 	}
+	c.StorageClass = strings.TrimSpace(c.StorageClass)
+	c.ACL = strings.TrimSpace(c.ACL)
 	return c.checkPartSizeAndConcurrency()
 }
 
@@ -329,6 +338,7 @@ func (c *GCSFsConfig) Validate(credentialsFilePath string) error {
 			return errors.New("credentials cannot be empty")
 		}
 	}
+	c.StorageClass = strings.TrimSpace(c.StorageClass)
 	return nil
 }