Bug 1632: Support S3 servers with Path URI style (like Minio)

https://winscp.net/tracker/1632

Source commit: 5bad75cc64d9c099d0736bcf2aec365235169418

source/core/S3FileSystem.cpp

@@ -472,6 +472,10 @@ TLibS3BucketContext TS3FileSystem::GetBucketContext(const UnicodeString & Bucket
     if (I != FRegions.end())
     {
       Region = I->second;
+      if (Region.IsEmpty())
+      {
+        Region = FAuthRegion;
+      }
     }
     else
     {
@@ -484,11 +488,20 @@ TLibS3BucketContext TS3FileSystem::GetBucketContext(const UnicodeString & Bucket
       Retry = true;
     }
 
+    S3UriStyle UriStyle = S3UriStyle(FTerminal->SessionData->S3UrlStyle);
+
     I = FHostNames.find(BucketName);
     UnicodeString HostName;
     if (I != FHostNames.end())
     {
       HostName = I->second;
+      if (SameText(HostName.SubString(1, BucketName.Length() + 1), BucketName + L"."))
+      {
+        HostName.Delete(1, BucketName.Length() + 1);
+        // Even when using path-style URL Amazon seems to redirect us to bucket hostname and
+        // we need to switch to virtual host style URL (without bucket name in the path)
+        UriStyle = S3UriStyleVirtualHost;
+      }
     }
     else
     {
@@ -500,7 +513,7 @@ TLibS3BucketContext TS3FileSystem::GetBucketContext(const UnicodeString & Bucket
     Result.BucketNameBuf = UTF8String(BucketName);
     Result.bucketName = Result.BucketNameBuf.c_str();
     Result.protocol = FLibS3Protocol;
-    Result.uriStyle = S3UriStyleVirtualHost;
+    Result.uriStyle = UriStyle;
     Result.accessKeyId = FAccessKeyId.c_str();
     Result.secretAccessKey = FSecretAccessKey.c_str();
     Result.securityToken = NULL;
@@ -530,10 +543,6 @@ TLibS3BucketContext TS3FileSystem::GetBucketContext(const UnicodeString & Bucket
                !Data.EndpointDetail.IsEmpty())
       {
         UnicodeString Endpoint = Data.EndpointDetail;
-        if (SameText(Endpoint.SubString(1, BucketName.Length() + 1), BucketName + L"."))
-        {
-          Endpoint.Delete(1, BucketName.Length() + 1);
-        }
         if (HostName != Endpoint)
         {
           FTerminal->LogEvent(FORMAT("Will use endpoint \"%s\" for bucket \"%s\" from now on.", (Endpoint, BucketName)));
@@ -541,6 +550,12 @@ TLibS3BucketContext TS3FileSystem::GetBucketContext(const UnicodeString & Bucket
           Retry = true;
         }
       }
+      // Minio
+      else if (Data.Status == S3StatusOK)
+      {
+        FTerminal->LogEvent(FORMAT("Will use default region for bucket \"%s\" from now on.", (BucketName)));
+        FRegions.insert(std::make_pair(BucketName, UnicodeString()));
+      }
     }
   }
   while (Retry);

source/core/SessionData.cpp

@@ -230,6 +230,7 @@ void __fastcall TSessionData::DefaultSettings()
   NotUtf = asAuto;
 
   S3DefaultRegion = L"";
+  S3UrlStyle = s3usVirtualHost;
 
   // SFTP
   SftpServer = L"";
@@ -383,6 +384,7 @@ void __fastcall TSessionData::NonPersistant()
   PROPERTY(PostLoginCommands); \
   \
   PROPERTY(S3DefaultRegion); \
+  PROPERTY(S3UrlStyle); \
   \
   PROPERTY(ProxyMethod); \
   PROPERTY(ProxyHost); \
@@ -708,6 +710,7 @@ void __fastcall TSessionData::DoLoad(THierarchicalStorage * Storage, bool PuttyI
   InternalEditorEncoding = Storage->ReadInteger(L"InternalEditorEncoding", InternalEditorEncoding);
 
   S3DefaultRegion = Storage->ReadString(L"S3DefaultRegion", S3DefaultRegion);
+  S3UrlStyle = (TS3UrlStyle)Storage->ReadInteger(L"S3UrlStyle", S3UrlStyle);
 
   // PuTTY defaults to TcpNoDelay, but the psftp/pscp ignores this preference, and always set this to off (what is our default too)
   if (!PuttyImport)
@@ -1081,6 +1084,7 @@ void __fastcall TSessionData::DoSave(THierarchicalStorage * Storage,
     WRITE_DATA_EX(Integer, L"Utf", NotUtf, );
     WRITE_DATA(Integer, InternalEditorEncoding);
     WRITE_DATA(String, S3DefaultRegion);
+    WRITE_DATA(Integer, S3UrlStyle);
     WRITE_DATA(Integer, SendBuf);
     WRITE_DATA(String, SourceAddress);
     WRITE_DATA(Bool, SshSimple);
@@ -3976,6 +3980,11 @@ void __fastcall TSessionData::SetS3DefaultRegion(UnicodeString value)
   SET_SESSION_PROPERTY(S3DefaultRegion);
 }
 //---------------------------------------------------------------------
+void __fastcall TSessionData::SetS3UrlStyle(TS3UrlStyle value)
+{
+  SET_SESSION_PROPERTY(S3UrlStyle);
+}
+//---------------------------------------------------------------------
 void __fastcall TSessionData::SetIsWorkspace(bool value)
 {
   SET_SESSION_PROPERTY(IsWorkspace);

source/core/SessionData.h

@@ -37,6 +37,8 @@ enum TFtps { ftpsNone, ftpsImplicit, ftpsExplicitSsl, ftpsExplicitTls };
 // has to match SSL_VERSION_XXX constants in AsyncSslSocketLayer.h
 // ssl2 has no effect now
 enum TTlsVersion { ssl2 = 2, ssl3 = 3, tls10 = 10, tls11 = 11, tls12 = 12 };
+// has to match libs3 S3UriStyle
+enum TS3UrlStyle { s3usVirtualHost, s3usPath };
 enum TSessionSource { ssNone, ssStored, ssStoredModified };
 enum TSessionUrlFlags
 {
@@ -218,6 +220,7 @@ private:
   TAutoSwitch FNotUtf;
   int FInternalEditorEncoding;
   UnicodeString FS3DefaultRegion;
+  TS3UrlStyle FS3UrlStyle;
   bool FIsWorkspace;
   UnicodeString FLink;
   UnicodeString FNameOverride;
@@ -392,6 +395,7 @@ private:
   void __fastcall SetNotUtf(TAutoSwitch value);
   void __fastcall SetInternalEditorEncoding(int value);
   void __fastcall SetS3DefaultRegion(UnicodeString value);
+  void __fastcall SetS3UrlStyle(TS3UrlStyle value);
   void __fastcall SetLogicalHostName(UnicodeString value);
   void __fastcall SetIsWorkspace(bool value);
   void __fastcall SetLink(UnicodeString value);
@@ -649,6 +653,7 @@ public:
   __property TAutoSwitch NotUtf = { read = FNotUtf, write = SetNotUtf };
   __property int InternalEditorEncoding = { read = FInternalEditorEncoding, write = SetInternalEditorEncoding };
   __property UnicodeString S3DefaultRegion = { read = FS3DefaultRegion, write = SetS3DefaultRegion };
+  __property TS3UrlStyle S3UrlStyle = { read = FS3UrlStyle, write = SetS3UrlStyle };
   __property bool IsWorkspace = { read = FIsWorkspace, write = SetIsWorkspace };
   __property UnicodeString Link = { read = FLink, write = SetLink };
   __property UnicodeString NameOverride = { read = FNameOverride, write = SetNameOverride };

source/core/SessionInfo.cpp

@@ -1352,6 +1352,7 @@ void __fastcall TSessionLog::DoAddStartupInfo(TSessionData * Data)
     {
       FtpsOn = (Data->Ftps != ftpsNone);
       ADF(L"HTTPS: %s", (BooleanToEngStr(FtpsOn)));
+      ADF(L"S3: URL Style: %s", (EnumName(Data->S3UrlStyle, L"Virtual Host;Path")));
       if (!Data->S3DefaultRegion.IsEmpty())
       {
         ADF(L"S3: Default region: %s", (Data->S3DefaultRegion));

source/forms/SiteAdvanced.cpp

@@ -215,6 +215,14 @@ void __fastcall TSiteAdvancedDialog::LoadSession()
 
     // S3 page
     S3DefaultReqionCombo->Text = FSessionData->S3DefaultRegion;
+    if (FSessionData->S3UrlStyle == s3usPath)
+    {
+      S3UrlStyleCombo->ItemIndex = 1;
+    }
+    else
+    {
+      S3UrlStyleCombo->ItemIndex = 0;
+    }
 
     // Authentication page
     SshNoUserAuthCheck->Checked = FSessionData->SshNoUserAuth;
@@ -601,6 +609,14 @@ void __fastcall TSiteAdvancedDialog::SaveSession()
 
   // S3 page
   FSessionData->S3DefaultRegion = S3DefaultReqionCombo->Text;
+  if (S3UrlStyleCombo->ItemIndex == 1)
+  {
+    FSessionData->S3UrlStyle = s3usPath;
+  }
+  else
+  {
+    FSessionData->S3UrlStyle = s3usVirtualHost;
+  }
 
   // Proxy page
   FSessionData->ProxyMethod = GetProxyMethod();

source/forms/SiteAdvanced.dfm

@@ -948,13 +948,13 @@ object SiteAdvancedDialog: TSiteAdvancedDialog
           Left = 0
           Top = 6
           Width = 393
-          Height = 46
+          Height = 70
           Anchors = [akLeft, akTop, akRight]
           Caption = 'Protocol options'
           TabOrder = 0
           DesignSize = (
             393
-            46)
+            70)
           object Label27: TLabel
             Left = 12
             Top = 20
@@ -963,6 +963,14 @@ object SiteAdvancedDialog: TSiteAdvancedDialog
             Caption = '&Default region:'
             FocusControl = S3DefaultReqionCombo
           end
+          object S3UrlStyleLabel: TLabel
+            Left = 12
+            Top = 44
+            Width = 50
+            Height = 13
+            Caption = '&URL Style:'
+            FocusControl = S3UrlStyleCombo
+          end
           object S3DefaultReqionCombo: TComboBox
             Left = 168
             Top = 15
@@ -998,6 +1006,20 @@ object SiteAdvancedDialog: TSiteAdvancedDialog
               'us-west-1'
               'us-west-2')
           end
+          object S3UrlStyleCombo: TComboBox
+            Left = 168
+            Top = 39
+            Width = 213
+            Height = 21
+            AutoComplete = False
+            Style = csDropDownList
+            Anchors = [akLeft, akTop, akRight]
+            MaxLength = 50
+            TabOrder = 1
+            Items.Strings = (
+              'Virtual Host'
+              'Path')
+          end
         end
       end
       object ConnSheet: TTabSheet

source/forms/SiteAdvanced.h

@@ -274,7 +274,7 @@ __published:
   TGroupBox *S3Group;
   TLabel *Label27;
   TComboBox *S3DefaultReqionCombo;
-  TLabel *Label5;
+  TLabel *S3UrlStyleLabel;
   TComboBox *S3UrlStyleCombo;
   void __fastcall DataChange(TObject *Sender);
   void __fastcall FormShow(TObject *Sender);