Bug 1921: Allow forcing use of LIST command to retrieve file information in scripting for FTP servers that has broken support for MDTM/SIZE commands

https://winscp.net/tracker/1921
(cherry picked from commit 7e78952092fe2c2a3010be44109696901119ff08)

Source commit: e2445f16f28468fc0fc6931c20ed2079a72409c1

source/core/Common.cpp

@@ -183,6 +183,7 @@ UnicodeString DefaultStr(const UnicodeString & Str, const UnicodeString & Defaul
   }
 }
 //---------------------------------------------------------------------------
+// For alternative with quoting support, see TFTPFileSystem::CutFeature
 UnicodeString CutToChar(UnicodeString &Str, wchar_t Ch, bool Trim)
 {
   Integer P = Str.Pos(Ch);

source/core/FtpFileSystem.cpp

@@ -199,6 +199,8 @@ const UnicodeString CopySiteCommand(L"COPY");
 const UnicodeString HashCommand(L"HASH"); // Cerberos + FileZilla servers
 const UnicodeString AvblCommand(L"AVBL");
 const UnicodeString XQuotaCommand(L"XQUOTA");
+const UnicodeString MdtmCommand(L"MDTM");
+const UnicodeString SizeCommand(L"SIZE");
 const UnicodeString DirectoryHasBytesPrefix(L"226-Directory has");
 //---------------------------------------------------------------------------
 class TFileListHelper
@@ -525,6 +527,8 @@ void __fastcall TFTPFileSystem::Open()
   }
   while (FPasswordFailed);
 
+  ProcessFeatures();
+
   // see also TWebDAVFileSystem::CollectTLSSessionInfo()
   FSessionInfo.CSCipher = FFileZillaIntf->GetCipherName().c_str();
   FSessionInfo.SCCipher = FSessionInfo.CSCipher;
@@ -2289,8 +2293,7 @@ bool __fastcall TFTPFileSystem::SupportsReadingFile()
 {
   return
     FFileZillaIntf->UsingMlsd() ||
-    ((FServerCapabilities->GetCapability(mdtm_command) == yes) &&
-     (FServerCapabilities->GetCapability(size_command) == yes));
+    (SupportsCommand(MdtmCommand) && SupportsCommand(SizeCommand));
 }
 //---------------------------------------------------------------------------
 void __fastcall TFTPFileSystem::ReadFile(const UnicodeString FileName,
@@ -3402,66 +3405,146 @@ void __fastcall TFTPFileSystem::ResetFeatures()
   FSupportsAnyChecksumFeature = false;
 }
 //---------------------------------------------------------------------------
-void __fastcall TFTPFileSystem::HandleFeatReply()
+UnicodeString TFTPFileSystem::CutFeature(UnicodeString & Buf)
 {
-  ResetFeatures();
-  // Response to FEAT must be multiline, where leading and trailing line
-  // is "meaningless". See RFC 2389.
-  if ((FLastCode == 211) && (FLastResponse->Count > 2))
+  UnicodeString Result;
+  if (Buf.SubString(1, 1) == L"\"")
   {
-    FLastResponse->Delete(0);
-    FLastResponse->Delete(FLastResponse->Count - 1);
-    FFeatures->Assign(FLastResponse);
+    Buf.Delete(1, 1);
+    int P = Buf.Pos(L"\",");
+    if (P == 0)
+    {
+      Result = Buf;
+      Buf = UnicodeString();
+      // there should be the ending quote, but if not, just do nothing
+      if (Result.SubString(Result.Length(), 1) == L"\"")
+      {
+        Result.SetLength(Result.Length() - 1);
+      }
+    }
+    else
+    {
+      Result = Buf.SubString(1, P - 1);
+      Buf.Delete(1, P + 1);
+    }
+    Buf = Buf.TrimLeft();
+  }
+  else
+  {
+    Result = CutToChar(Buf, L',', true);
+  }
+  return Result;
+}
+//---------------------------------------------------------------------------
+void TFTPFileSystem::ProcessFeatures()
+{
+  std::unique_ptr<TStrings> Features(new TStringList());
+  UnicodeString FeaturesOverride = FTerminal->SessionData->ProtocolFeatures.Trim();
+  if (FeaturesOverride.SubString(1, 1) == L"*")
+  {
+    FeaturesOverride.Delete(1, 1);
+    while (!FeaturesOverride.IsEmpty())
+    {
+      UnicodeString Feature = CutFeature(FeaturesOverride);
+      Features->Add(Feature);
+    }
+  }
+  else
+  {
+    std::unique_ptr<TStrings> DeleteFeatures(CreateSortedStringList());
+    std::unique_ptr<TStrings> AddFeatures(new TStringList());
+    while (!FeaturesOverride.IsEmpty())
+    {
+      UnicodeString Feature = CutFeature(FeaturesOverride);
+      if (Feature.SubString(1, 1) == L"-")
+      {
+        Feature.Delete(1, 1);
+        DeleteFeatures->Add(Feature.LowerCase());
+      }
+      else
+      {
+        if (Feature.SubString(1, 1) == L"+")
+        {
+          Feature.Delete(1, 1);
+        }
+        AddFeatures->Add(Feature);
+      }
+    }
+
     for (int Index = 0; Index < FFeatures->Count; Index++)
     {
       // IIS 2003 indents response by 4 spaces, instead of one,
       // see example in HandleReplyStatus
-      UnicodeString Feature = TrimLeft(FFeatures->Strings[Index]);
+      UnicodeString Feature = FFeatures->Strings[Index].Trim();
+      if (DeleteFeatures->IndexOf(Feature) < 0)
+      {
+        Features->Add(Feature);
+      }
+    }
 
-      UnicodeString Args = Feature;
-      UnicodeString Command = CutToChar(Args, L' ', true);
+    Features->AddStrings(AddFeatures.get());
+  }
+
+  for (int Index = 0; Index < Features->Count; Index++)
+  {
+    UnicodeString Feature = Features->Strings[Index];
 
-      // Serv-U lists Xalg commands like:
-      //  XSHA1 filename;start;end
-      FSupportedCommands->Add(Command);
+    UnicodeString Args = Feature;
+    UnicodeString Command = CutToChar(Args, L' ', true);
 
-      if (SameText(Command, SiteCommand))
+    // Serv-U lists Xalg commands like:
+    //  XSHA1 filename;start;end
+    FSupportedCommands->Add(Command);
+
+    if (SameText(Command, SiteCommand))
+    {
+      // Serv-U lists all SITE commands in one line like:
+      //  SITE PSWD;SET;ZONE;CHMOD;MSG;EXEC;HELP
+      // But ProFTPD lists them separatelly:
+      //  SITE UTIME
+      //  SITE RMDIR
+      //  SITE COPY
+      //  SITE MKDIR
+      //  SITE SYMLINK
+      while (!Args.IsEmpty())
       {
-        // Serv-U lists all SITE commands in one line like:
-        //  SITE PSWD;SET;ZONE;CHMOD;MSG;EXEC;HELP
-        // But ProFTPD lists them separatelly:
-        //  SITE UTIME
-        //  SITE RMDIR
-        //  SITE COPY
-        //  SITE MKDIR
-        //  SITE SYMLINK
-        while (!Args.IsEmpty())
-        {
-          UnicodeString Arg = CutToChar(Args, L';', true);
-          FSupportedSiteCommands->Add(Arg);
-        }
+        UnicodeString Arg = CutToChar(Args, L';', true);
+        FSupportedSiteCommands->Add(Arg);
       }
-      else if (SameText(Command, HashCommand))
+    }
+    else if (SameText(Command, HashCommand))
+    {
+      while (!Args.IsEmpty())
       {
-        while (!Args.IsEmpty())
+        UnicodeString Alg = CutToChar(Args, L';', true);
+        if ((Alg.Length() > 0) && (Alg[Alg.Length()] == L'*'))
         {
-          UnicodeString Alg = CutToChar(Args, L';', true);
-          if ((Alg.Length() > 0) && (Alg[Alg.Length()] == L'*'))
-          {
-            Alg.Delete(Alg.Length(), 1);
-          }
-          // FTP HASH alg names follow IANA as we do,
-          // but using uppercase and we use lowercase
-          FHashAlgs->Add(LowerCase(Alg));
-          FSupportsAnyChecksumFeature = true;
+          Alg.Delete(Alg.Length(), 1);
         }
-      }
-
-      if (FChecksumCommands->IndexOf(Command) >= 0)
-      {
+        // FTP HASH alg names follow IANA as we do,
+        // but using uppercase and we use lowercase
+        FHashAlgs->Add(LowerCase(Alg));
         FSupportsAnyChecksumFeature = true;
       }
     }
+
+    if (FChecksumCommands->IndexOf(Command) >= 0)
+    {
+      FSupportsAnyChecksumFeature = true;
+    }
+  }
+}
+//---------------------------------------------------------------------------
+void __fastcall TFTPFileSystem::HandleFeatReply()
+{
+  ResetFeatures();
+  // Response to FEAT must be multiline, where leading and trailing line
+  // is "meaningless". See RFC 2389.
+  if ((FLastCode == 211) && (FLastResponse->Count > 2))
+  {
+    FLastResponse->Delete(0);
+    FLastResponse->Delete(FLastResponse->Count - 1);
+    FFeatures->Assign(FLastResponse);
   }
 }
 //---------------------------------------------------------------------------

source/core/FtpFileSystem.h

@@ -204,6 +204,8 @@ protected:
     TFileOperationProgressType * OperationProgress, bool FirstLevel);
   void __fastcall HandleFeatReply();
   void __fastcall ResetFeatures();
+  void ProcessFeatures();
+  UnicodeString CutFeature(UnicodeString & Buf);
   bool __fastcall SupportsSiteCommand(const UnicodeString & Command) const;
   bool __fastcall SupportsCommand(const UnicodeString & Command) const;
   void __fastcall RegisterChecksumAlgCommand(const UnicodeString & Alg, const UnicodeString & Command);

source/core/SessionData.cpp

@@ -164,6 +164,7 @@ void __fastcall TSessionData::DefaultSettings()
   TcpNoDelay = false;
   SendBuf = DefaultSendBuf;
   SourceAddress = L"";
+  ProtocolFeatures = L"";
   SshSimple = true;
   HostKey = L"";
   FingerprintScan = false;
@@ -378,6 +379,7 @@ void __fastcall TSessionData::NonPersistant()
   PROPERTY(TcpNoDelay); \
   PROPERTY(SendBuf); \
   PROPERTY(SourceAddress); \
+  PROPERTY(ProtocolFeatures); \
   PROPERTY(SshSimple); \
   PROPERTY(AuthKI); \
   PROPERTY(AuthKIPassword); \
@@ -733,6 +735,7 @@ void __fastcall TSessionData::DoLoad(THierarchicalStorage * Storage, bool PuttyI
   }
   SendBuf = Storage->ReadInteger(L"SendBuf", Storage->ReadInteger("SshSendBuf", SendBuf));
   SourceAddress = Storage->ReadString(L"SourceAddress", SourceAddress);
+  ProtocolFeatures = Storage->ReadString(L"ProtocolFeatures", ProtocolFeatures);
   SshSimple = Storage->ReadBool(L"SshSimple", SshSimple);
 
   ProxyMethod = (TProxyMethod)Storage->ReadInteger(L"ProxyMethod", ProxyMethod);
@@ -1107,6 +1110,7 @@ void __fastcall TSessionData::DoSave(THierarchicalStorage * Storage,
     WRITE_DATA(Integer, S3UrlStyle);
     WRITE_DATA(Integer, SendBuf);
     WRITE_DATA(String, SourceAddress);
+    WRITE_DATA(String, ProtocolFeatures);
     WRITE_DATA(Bool, SshSimple);
   }
 
@@ -3753,6 +3757,11 @@ void __fastcall TSessionData::SetSourceAddress(const UnicodeString & value)
   SET_SESSION_PROPERTY(SourceAddress);
 }
 //---------------------------------------------------------------------
+void __fastcall TSessionData::SetProtocolFeatures(const UnicodeString & value)
+{
+  SET_SESSION_PROPERTY(ProtocolFeatures);
+}
+//---------------------------------------------------------------------
 void __fastcall TSessionData::SetSshSimple(bool value)
 {
   SET_SESSION_PROPERTY(SshSimple);

source/core/SessionData.h

@@ -159,6 +159,7 @@ private:
   bool FTcpNoDelay;
   int FSendBuf;
   UnicodeString FSourceAddress;
+  UnicodeString FProtocolFeatures;
   bool FSshSimple;
   TProxyMethod FProxyMethod;
   UnicodeString FProxyHost;
@@ -322,6 +323,7 @@ private:
   void __fastcall SetTcpNoDelay(bool value);
   void __fastcall SetSendBuf(int value);
   void __fastcall SetSourceAddress(const UnicodeString & value);
+  void __fastcall SetProtocolFeatures(const UnicodeString & value);
   void __fastcall SetSshSimple(bool value);
   UnicodeString __fastcall GetSshProtStr();
   bool __fastcall GetUsesSsh();
@@ -596,6 +598,7 @@ public:
   __property bool TcpNoDelay  = { read=FTcpNoDelay, write=SetTcpNoDelay };
   __property int SendBuf  = { read=FSendBuf, write=SetSendBuf };
   __property UnicodeString SourceAddress = { read=FSourceAddress, write=SetSourceAddress };
+  __property UnicodeString ProtocolFeatures = { read=FProtocolFeatures, write=SetProtocolFeatures };
   __property bool SshSimple  = { read=FSshSimple, write=SetSshSimple };
   __property UnicodeString SshProtStr  = { read=GetSshProtStr };
   __property UnicodeString CipherList  = { read=GetCipherList, write=SetCipherList };

source/filezilla/FileZillaIntf.h

@@ -251,7 +251,6 @@ enum ftp_capability_names_t
   mfmt_command,
   pret_command,
   mdtm_command,
-  size_command,
   mode_z_support,
   tvfs_support, // Trivial virtual file store (RFC 3659)
   list_hidden_support, // LIST -a command

source/filezilla/FtpControlSocket.cpp

@@ -6102,10 +6102,6 @@ void CFtpControlSocket::DiscardLine(CStringA line)
     {
       m_serverCapabilities.SetCapability(mdtm_command, yes);
     }
-    else if (line == "SIZE")
-    {
-      m_serverCapabilities.SetCapability(size_command, yes);
-    }
     else if (line.Left(4) == "MLST")
     {
       USES_CONVERSION;