Browse Source

S3: fix Ceph compatibility

This hack will no longer be needed once Ceph tags a new version and vendors
using it update their servers.

This code is taken from rclone, thank you!

Fixes #483
Nicola Murino 4 years ago
parent
commit
c997ef876c
1 changed files with 20 additions and 1 deletions
  1. 20 1
      vfs/s3fs.go

+ 20 - 1
vfs/s3fs.go

@@ -290,11 +290,19 @@ func (fs *S3Fs) Rename(source, target string) error {
 	defer cancelFn()
 	_, err = fs.svc.CopyObjectWithContext(ctx, &s3.CopyObjectInput{
 		Bucket:       aws.String(fs.config.Bucket),
-		CopySource:   aws.String(url.PathEscape(copySource)),
+		CopySource:   aws.String(pathEscape(copySource)),
 		Key:          aws.String(target),
 		StorageClass: util.NilIfEmpty(fs.config.StorageClass),
 		ContentType:  util.NilIfEmpty(contentType),
 	})
+	if err != nil {
+		metric.S3CopyObjectCompleted(err)
+		return err
+	}
+	err = fs.svc.WaitUntilObjectExistsWithContext(ctx, &s3.HeadObjectInput{
+		Bucket: aws.String(fs.config.Bucket),
+		Key:    aws.String(target),
+	})
 	metric.S3CopyObjectCompleted(err)
 	if err != nil {
 		return err
@@ -698,3 +706,14 @@ func (*S3Fs) Close() error {
 func (*S3Fs) GetAvailableDiskSize(dirName string) (*sftp.StatVFS, error) {
 	return nil, ErrStorageSizeUnavailable
 }
+
+// ideally we should simply use url.PathEscape:
+//
+// https://github.com/awsdocs/aws-doc-sdk-examples/blob/master/go/example_code/s3/s3_copy_object.go#L65
+//
+// but this cause issue with some vendors, see #483, the code below is copied from rclone
+func pathEscape(in string) string {
+	var u url.URL
+	u.Path = in
+	return strings.ReplaceAll(u.String(), "+", "%2B")
+}