Bug 1945: Streaming support in .NET assembly for FTP protocol

https://winscp.net/tracker/1945
(cherry picked from commit 9370bfd17f7c749ccc8ff53ffb01efc10910daf1)

Source commit: 9602b495c733729c7e1c180cb05001f5d5af4104

source/core/FtpFileSystem.cpp

@@ -1581,8 +1581,9 @@ void __fastcall TFTPFileSystem::FileTransfer(const UnicodeString & FileName,
 {
   FILE_OPERATION_LOOP_BEGIN
   {
-    FFileZillaIntf->FileTransfer(ApiPath(LocalFile).c_str(), RemoteFile.c_str(),
-      RemotePath.c_str(), Get, Size, Type, &UserData);
+    FFileZillaIntf->FileTransfer(
+      ApiPath(LocalFile).c_str(), RemoteFile.c_str(), RemotePath.c_str(),
+      Get, Size, Type, &UserData, UserData.CopyParam->OnTransferOut, UserData.CopyParam->OnTransferIn);
     // we may actually catch response code of the listing
     // command (when checking for existence of the remote file)
     unsigned int Reply = WaitForCommandReply();
@@ -1694,7 +1695,10 @@ void __fastcall TFTPFileSystem::Sink(
   UnicodeString ExpandedDestFullName = ExpandUNCFileName(DestFullName);
   Action.Destination(ExpandedDestFullName);
 
-  FTerminal->UpdateTargetAttrs(DestFullName, File, CopyParam, Attrs);
+  if (CopyParam->OnTransferOut == NULL)
+  {
+    FTerminal->UpdateTargetAttrs(DestFullName, File, CopyParam, Attrs);
+  }
 
   FLastDataSent = Now();
 }
@@ -1718,12 +1722,13 @@ void __fastcall TFTPFileSystem::CopyToRemote(TStrings * FilesToCopy,
 bool __fastcall TFTPFileSystem::CanTransferSkipList(int Params, unsigned int Flags, const TCopyParamType * CopyParam)
 {
   bool Result =
-    FLAGSET(Params, cpNoConfirmation) &&
-    // cpAppend is not supported with FTP
-    DebugAlwaysTrue(FLAGCLEAR(Params, cpAppend)) &&
-    FLAGCLEAR(Params, cpResume) &&
-    FLAGCLEAR(Flags, tfAutoResume) &&
-    !CopyParam->NewerOnly;
+    (CopyParam->OnTransferIn != NULL) ||
+    (FLAGSET(Params, cpNoConfirmation) &&
+     // cpAppend is not supported with FTP
+     DebugAlwaysTrue(FLAGCLEAR(Params, cpAppend)) &&
+     FLAGCLEAR(Params, cpResume) &&
+     FLAGCLEAR(Flags, tfAutoResume) &&
+     !CopyParam->NewerOnly);
   return Result;
 }
 //---------------------------------------------------------------------------
@@ -1733,7 +1738,10 @@ void __fastcall TFTPFileSystem::Source(
   TFileOperationProgressType * OperationProgress, unsigned int Flags,
   TUploadSessionAction & Action, bool & /*ChildError*/)
 {
-  Handle.Close();
+  if (CopyParam->OnTransferIn == NULL)
+  {
+    Handle.Close();
+  }
 
   ResetFileTransfer();
 
@@ -1749,8 +1757,9 @@ void __fastcall TFTPFileSystem::Source(
 
     SetCPSLimit(OperationProgress);
     // not used for uploads anyway
-    FFileTransferPreserveTime = CopyParam->PreserveTime;
+    FFileTransferPreserveTime = CopyParam->PreserveTime && (CopyParam->OnTransferIn == NULL);
     FFileTransferRemoveBOM = CopyParam->RemoveBOM;
+    // not used for uploads anyway
     FFileTransferNoList = CanTransferSkipList(Params, Flags, CopyParam);
     // not used for uploads, but we get new name (if any) back in this field
     UserData.FileName = DestFileName;
@@ -1927,6 +1936,8 @@ bool __fastcall TFTPFileSystem::IsCapable(int Capability) const
     case fcMoveToQueue:
     case fcSkipTransfer:
     case fcParallelTransfers:
+    case fcTransferOut:
+    case fcTransferIn:
       return true;
 
     case fcPreservingTimestampUpload:
@@ -1962,8 +1973,6 @@ bool __fastcall TFTPFileSystem::IsCapable(int Capability) const
     case fcPreservingTimestampDirs:
     case fcResumeSupport:
     case fcChangePassword:
-    case fcTransferOut:
-    case fcTransferIn:
       return false;
 
     default:
@@ -3877,6 +3886,11 @@ bool __fastcall TFTPFileSystem::HandleAsynchRequestOverwrite(
       // on retry, use the same answer as on the first attempt
       RequestResult = UserData.OverwriteResult;
     }
+    else if ((UserData.CopyParam->OnTransferOut != NULL) || (UserData.CopyParam->OnTransferIn != NULL))
+    {
+      DebugFail();
+      RequestResult = TFileZillaIntf::FILEEXISTS_OVERWRITE;
+    }
     else
     {
       TFileOperationProgressType * OperationProgress = FTerminal->OperationProgress;
@@ -4684,6 +4698,7 @@ bool __fastcall TFTPFileSystem::Unquote(UnicodeString & Str)
 void __fastcall TFTPFileSystem::PreserveDownloadFileTime(HANDLE Handle, void * UserData)
 {
   TFileTransferData * Data = static_cast<TFileTransferData *>(UserData);
+  DebugAssert(Data->CopyParam->OnTransferOut == NULL);
   FTerminal->UpdateTargetTime(Handle, Data->Modification, dstmUnix);
 }
 //---------------------------------------------------------------------------

source/filezilla/FileZillaIntf.cpp

@@ -218,9 +218,10 @@ bool __fastcall TFileZillaIntf::ListFile(const wchar_t * FileName, const wchar_t
   return Check(FFileZillaApi->ListFile(FileName, Path), L"listfile");
 }
 //---------------------------------------------------------------------------
-bool __fastcall TFileZillaIntf::FileTransfer(const wchar_t * LocalFile,
-  const wchar_t * RemoteFile, const wchar_t * RemotePath, bool Get, __int64 Size,
-  int Type, void * UserData)
+bool __fastcall TFileZillaIntf::FileTransfer(
+  const wchar_t * LocalFile, const wchar_t * RemoteFile,
+  const wchar_t * RemotePath, bool Get, __int64 Size, int Type, void * UserData,
+  TTransferOutEvent OnTransferOut, TTransferInEvent OnTransferIn)
 {
   t_transferfile Transfer;
 
@@ -233,6 +234,8 @@ bool __fastcall TFileZillaIntf::FileTransfer(const wchar_t * LocalFile,
   // 1 = ascii, 2 = binary
   Transfer.nType = Type;
   Transfer.nUserData = reinterpret_cast<int>(UserData);
+  Transfer.OnTransferOut = OnTransferOut;
+  Transfer.OnTransferIn = OnTransferIn;
 
   return Check(FFileZillaApi->FileTransfer(Transfer), L"filetransfer");
 }

source/filezilla/FileZillaIntf.h

@@ -195,8 +195,10 @@ public:
   bool __fastcall Rename(const wchar_t* OldName, const wchar_t* NewName,
     const wchar_t* Path, const wchar_t* NewPath);
 
-  bool __fastcall FileTransfer(const wchar_t * LocalFile, const wchar_t * RemoteFile,
-    const wchar_t * RemotePath, bool Get, __int64 Size, int Type, void * UserData);
+  bool __fastcall FileTransfer(
+    const wchar_t * LocalFile, const wchar_t * RemoteFile,
+    const wchar_t * RemotePath, bool Get, __int64 Size, int Type, void * UserData,
+    TTransferOutEvent OnTransferOut, TTransferInEvent OnTransferIn);
 
   virtual const wchar_t * __fastcall Option(int OptionID) const = 0;
   virtual int __fastcall OptionVal(int OptionID) const = 0;

source/filezilla/FtpControlSocket.cpp

@@ -2841,10 +2841,14 @@ void CFtpControlSocket::FileTransfer(t_transferfile *transferfile/*=0*/,BOOL bFi
     DebugAssert(!m_Operation.nOpMode);
     DebugAssert(!m_Operation.pData);
 
-    CString str;
-    str.Format(transferfile->get?IDS_STATUSMSG_DOWNLOADSTART:IDS_STATUSMSG_UPLOADSTART,
-          transferfile->get ? transferfile->remotepath.FormatFilename(transferfile->remotefile) : transferfile->localfile);
-    ShowStatus(str,FZ_LOG_STATUS);
+    if ((transferfile->OnTransferOut == NULL) &&
+        (transferfile->OnTransferIn == NULL))
+    {
+      CString str;
+      str.Format(transferfile->get?IDS_STATUSMSG_DOWNLOADSTART:IDS_STATUSMSG_UPLOADSTART,
+            transferfile->get ? transferfile->remotepath.FormatFilename(transferfile->remotefile) : transferfile->localfile);
+      ShowStatus(str,FZ_LOG_STATUS);
+    }
 
     m_Operation.nOpMode=CSMODE_TRANSFER|(transferfile->get?CSMODE_DOWNLOAD:CSMODE_UPLOAD);
 
@@ -3502,18 +3506,31 @@ void CFtpControlSocket::FileTransfer(t_transferfile *transferfile/*=0*/,BOOL bFi
           m_Operation.nOpState = FILETRANSFER_REST;
         else
           m_Operation.nOpState = FILETRANSFER_RETRSTOR;
-        BOOL res = FALSE;
-        if (!m_pDataFile)
-          m_pDataFile = new CFile;
+        BOOL res;
+        if (m_pDataFile != NULL)
+        {
+          delete m_pDataFile;
+          m_pDataFile = NULL;
+        }
         if (pData->transferfile.get)
         {
-          if (pData->transferdata.bResume)
-            res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeCreate|CFile::modeWrite|CFile::modeNoTruncate|CFile::shareDenyWrite);
-          else
-            res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeWrite|CFile::modeCreate|CFile::shareDenyWrite);
+          if (pData->transferfile.OnTransferOut == NULL)
+          {
+            m_pDataFile = new CFile();
+            if (pData->transferdata.bResume)
+              res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeCreate|CFile::modeWrite|CFile::modeNoTruncate|CFile::shareDenyWrite);
+            else
+              res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeWrite|CFile::modeCreate|CFile::shareDenyWrite);
+          }
         }
         else
-          res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeRead|CFile::shareDenyNone);
+        {
+          if (pData->transferfile.OnTransferIn == NULL)
+          {
+            m_pDataFile = new CFile();
+            res = m_pDataFile->Open(pData->transferfile.localfile,CFile::modeRead|CFile::shareDenyNone);
+          }
+        }
         if (!res)
         {
           wchar_t * Error = m_pTools->LastSysErrorMessage();
@@ -3535,11 +3552,16 @@ void CFtpControlSocket::FileTransfer(t_transferfile *transferfile/*=0*/,BOOL bFi
         }
 
         m_pTransferSocket->m_pFile = m_pDataFile;
+        m_pTransferSocket->m_OnTransferOut = pData->transferfile.OnTransferOut;
+        m_pTransferSocket->m_OnTransferIn = pData->transferfile.OnTransferIn;
         if (!pData->transferfile.get)
         {
-          // See comment in !get branch below
-          pData->transferdata.transfersize=GetLength64(*m_pDataFile);
-          pData->transferdata.transferleft=pData->transferdata.transfersize;
+          if (m_pDataFile != NULL)
+          {
+            // See comment in !get branch below
+            pData->transferdata.transfersize=GetLength64(*m_pDataFile);
+            pData->transferdata.transferleft=pData->transferdata.transfersize;
+          }
           if (pData->transferdata.bResume)
           {
             CString remotefile=pData->transferfile.remotefile;
@@ -4385,8 +4407,10 @@ void CFtpControlSocket::TransferFinished(bool preserveFileTimeForUploads)
 {
   CFileTransferData *pData=static_cast<CFileTransferData *>(m_Operation.pData);
 
-  if (GetOptionVal(OPTION_PRESERVEDOWNLOADFILETIME) && m_pDataFile &&
-        pData->transferfile.get)
+  if (GetOptionVal(OPTION_PRESERVEDOWNLOADFILETIME) &&
+      m_pDataFile &&
+      pData->transferfile.get &&
+      DebugAlwaysTrue(pData->transferfile.OnTransferOut == NULL))
   {
     m_pTools->PreserveDownloadFileTime(
       (HANDLE)m_pDataFile->m_hFile, reinterpret_cast<void *>(pData->transferfile.nUserData));
@@ -4394,7 +4418,8 @@ void CFtpControlSocket::TransferFinished(bool preserveFileTimeForUploads)
   if (!pData->transferfile.get &&
       GetOptionVal(OPTION_MPEXT_PRESERVEUPLOADFILETIME) && preserveFileTimeForUploads &&
       ((m_serverCapabilities.GetCapability(mfmt_command) == yes) ||
-       (m_serverCapabilities.GetCapability(mdtm_command) == yes)))
+       (m_serverCapabilities.GetCapability(mdtm_command) == yes)) &&
+      DebugAlwaysTrue(pData->transferfile.OnTransferIn == NULL))
   {
     CString filename =
       pData->transferfile.remotepath.FormatFilename(pData->transferfile.remotefile, !pData->bUseAbsolutePaths);
@@ -4867,7 +4892,7 @@ int CFtpControlSocket::CheckOverwriteFileAndCreateTarget()
   if (!nReplyError)
   {
     CFileTransferData * pData = static_cast<CFileTransferData *>(m_Operation.pData);
-    if (pData->transferfile.get)
+    if (pData->transferfile.get && (pData->transferfile.OnTransferOut == NULL))
     {
       CString path = pData->transferfile.localfile;
       if (path.ReverseFind(L'\\') != -1)
@@ -4897,8 +4922,12 @@ int CFtpControlSocket::CheckOverwriteFile()
 
   int nReplyError = 0;
   CFileStatus64 status;
-  BOOL res = GetStatus64(pData->transferfile.localfile, status);
-  if (!res)
+  if ((pData->transferfile.OnTransferOut != NULL) ||
+      (pData->transferfile.OnTransferIn != NULL))
+  {
+    m_Operation.nOpState = FILETRANSFER_TYPE;
+  }
+  else if (!GetStatus64(pData->transferfile.localfile, status))
   {
     if (!pData->transferfile.get)
     {

source/filezilla/FzApiStructures.h

@@ -3,6 +3,7 @@
 #define FzApiStructuresH
 //---------------------------------------------------------------------------
 #include <openssl/pkcs12.h>
+#include <FileBuffer.h>
 //---------------------------------------------------------------------------
 class t_server
 {
@@ -35,6 +36,8 @@ typedef struct
     t_server server;
     int nType;
     int nUserData;
+    TTransferOutEvent OnTransferOut;
+    TTransferInEvent OnTransferIn;
 } t_transferfile;
 //---------------------------------------------------------------------------
 #endif // FzApiStructuresH

source/filezilla/TransferSocket.cpp

@@ -30,6 +30,8 @@ CTransferSocket::CTransferSocket(CFtpControlSocket *pOwner, int nMode)
 #endif
   m_bufferpos = 0;
   m_pFile = 0;
+  m_OnTransferOut = NULL;
+  m_OnTransferIn = NULL;
   m_bListening = FALSE;
   m_bSentClose = FALSE;
   m_nInternalMessageID = 0;
@@ -256,7 +258,7 @@ void CTransferSocket::OnReceive(int nErrorCode)
         int res = inflate(&m_zlibStream, 0);
         while (res == Z_OK)
         {
-          m_pFile->Write(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
+          WriteData(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
           written += BUFSIZE - m_zlibStream.avail_out;
           m_zlibStream.next_out = (Bytef *)m_pBuffer2;
           m_zlibStream.avail_out = BUFSIZE;
@@ -264,7 +266,7 @@ void CTransferSocket::OnReceive(int nErrorCode)
         }
         if (res == Z_STREAM_END)
         {
-          m_pFile->Write(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
+          WriteData(m_pBuffer2, BUFSIZE - m_zlibStream.avail_out);
           written += BUFSIZE - m_zlibStream.avail_out;
         }
         else if (res != Z_OK && res != Z_BUF_ERROR)
@@ -277,7 +279,7 @@ void CTransferSocket::OnReceive(int nErrorCode)
       else
 #endif
       {
-        m_pFile->Write(m_pBuffer, numread);
+        WriteData(m_pBuffer, numread);
         written = numread;
       }
     }
@@ -689,7 +691,7 @@ void CTransferSocket::OnSend(int nErrorCode)
   else
 #endif
   {
-    if (!m_pFile)
+    if (!m_pFile && (m_OnTransferIn == NULL))
     {
       return;
     }
@@ -1098,6 +1100,33 @@ bool CTransferSocket::InitZlib(int level)
 }
 #endif
 
+void CTransferSocket::WriteData(const char * buffer, int len)
+{
+  if (m_OnTransferOut != NULL)
+  {
+    m_OnTransferOut(NULL, m_pBuffer, len);
+  }
+  else
+  {
+    m_pFile->Write(m_pBuffer, len);
+  }
+}
+
+int CTransferSocket::ReadData(char * buffer, int len)
+{
+  int result;
+  if (m_OnTransferIn != NULL)
+  {
+    result = m_OnTransferIn(NULL, buffer, len);
+  }
+  else
+  {
+    result = m_pFile->Read(buffer, len);
+  }
+  LogMessage(FZ_LOG_INFO, L"Read %d bytes from file", result);
+  return result;
+}
+
 int CTransferSocket::ReadDataFromFile(char *buffer, int len)
 {
   TRY
@@ -1105,13 +1134,13 @@ int CTransferSocket::ReadDataFromFile(char *buffer, int len)
     // Comparing to Filezilla 2, we do not do any translation locally,
     // leaving it onto the server (what Filezilla 3 seems to do too)
     const char Bom[3] = "\xEF\xBB\xBF";
-    int read = m_pFile->Read(buffer, len);
+    int read = ReadData(buffer, len);
     if (GetOptionVal(OPTION_MPEXT_REMOVE_BOM) &&
         m_transferdata.bType && (read >= sizeof(Bom)) && (memcmp(buffer, Bom, sizeof(Bom)) == 0))
     {
       memcpy(buffer, buffer + sizeof(Bom), read - sizeof(Bom));
       read -= sizeof(Bom);
-      int read2 = m_pFile->Read(buffer + read, sizeof(Bom));
+      int read2 = ReadData(buffer + read, sizeof(Bom));
       if (read2 > 0)
       {
         read += read2;

source/filezilla/TransferSocket.h

@@ -33,6 +33,8 @@ public:
   virtual BOOL Create(BOOL bUseSsl);
   BOOL m_bListening;
   CFile * m_pFile;
+  TTransferOutEvent m_OnTransferOut;
+  TTransferInEvent m_OnTransferIn;
   t_transferdata m_transferdata;
   __int64 m_uploaded;
   void SetActive();
@@ -55,6 +57,8 @@ public:
 protected:
   virtual int OnLayerCallback(std::list<t_callbackMsg> & callbacks);
   int ReadDataFromFile(char * buffer, int len);
+  int ReadData(char * buffer, int len);
+  void WriteData(const char * buffer, int len);
   virtual void LogSocketMessageRaw(int nMessageType, LPCTSTR pMsg);
   virtual int GetSocketOptionVal(int OptionID) const;
   virtual void ConfigureSocket();

source/windows/ConsoleRunner.cpp

@@ -560,6 +560,8 @@ private:
   bool FWantsProgress;
   bool FInteractive;
   unsigned int FMaxSend;
+  // Particularly FTP calls TransferOut/In from other thread
+  std::unique_ptr<TCriticalSection> FSection;
 
   inline TConsoleCommStruct * __fastcall GetCommStruct();
   inline void __fastcall FreeCommStruct(TConsoleCommStruct * CommStruct);
@@ -592,6 +594,8 @@ __fastcall TExternalConsole::TExternalConsole(
     CloseHandle(Job);
   }
 
+  FSection.reset(new TCriticalSection());
+
   TConsoleCommStruct * CommStruct = GetCommStruct();
   try
   {
@@ -686,6 +690,7 @@ UnicodeString __fastcall TExternalConsole::FinalLogMessage()
 //---------------------------------------------------------------------------
 void __fastcall TExternalConsole::Print(UnicodeString Str, bool FromBeginning, bool Error)
 {
+  TGuard Guard(FSection.get());
   // need to do at least one iteration, even when Str is empty (new line)
   do
   {
@@ -721,6 +726,7 @@ void __fastcall TExternalConsole::Print(UnicodeString Str, bool FromBeginning, b
 //---------------------------------------------------------------------------
 bool __fastcall TExternalConsole::Input(UnicodeString & Str, bool Echo, unsigned int Timer)
 {
+  TGuard Guard(FSection.get());
   TConsoleCommStruct * CommStruct = GetCommStruct();
   try
   {
@@ -757,6 +763,7 @@ int __fastcall TExternalConsole::Choice(
   UnicodeString Options, int Cancel, int Break, int Continue, int Timeouted, bool Timeouting, unsigned int Timer,
   UnicodeString Message)
 {
+  TGuard Guard(FSection.get());
   TConsoleCommStruct * CommStruct = GetCommStruct();
   try
   {
@@ -803,6 +810,7 @@ bool __fastcall TExternalConsole::PendingAbort()
 //---------------------------------------------------------------------------
 void __fastcall TExternalConsole::SetTitle(UnicodeString Title)
 {
+  TGuard Guard(FSection.get());
   TConsoleCommStruct * CommStruct = GetCommStruct();
   try
   {
@@ -822,6 +830,7 @@ void __fastcall TExternalConsole::SetTitle(UnicodeString Title)
 //---------------------------------------------------------------------------
 void __fastcall TExternalConsole::Init()
 {
+  TGuard Guard(FSection.get());
   TConsoleCommStruct * CommStruct = GetCommStruct();
   try
   {
@@ -897,6 +906,7 @@ void __fastcall TExternalConsole::WaitBeforeExit()
 //---------------------------------------------------------------------------
 void __fastcall TExternalConsole::Progress(TScriptProgress & Progress)
 {
+  TGuard Guard(FSection.get());
   TConsoleCommStruct * CommStruct = GetCommStruct();
 
   typedef TConsoleCommStruct::TProgressEvent TProgressEvent;
@@ -957,6 +967,7 @@ void __fastcall TExternalConsole::Progress(TScriptProgress & Progress)
 //---------------------------------------------------------------------------
 void __fastcall TExternalConsole::TransferOut(const unsigned char * Data, size_t Len)
 {
+  TGuard Guard(FSection.get());
   DebugAssert((Data == NULL) == (Len == 0));
   size_t Offset = 0;
   do
@@ -981,6 +992,7 @@ void __fastcall TExternalConsole::TransferOut(const unsigned char * Data, size_t
 //---------------------------------------------------------------------------
 size_t __fastcall TExternalConsole::TransferIn(unsigned char * Data, size_t Len)
 {
+  TGuard Guard(FSection.get());
   size_t Offset = 0;
   size_t Result = 0;
   while ((Result == Offset) && (Offset < Len))