Bug 875: Parallel background downloads

https://winscp.net/tracker/875

Source commit: 86d27203e082bbd4b4d5edb566d48d2697be2db4

source/core/Queue.cpp

@@ -12,10 +12,10 @@
 //---------------------------------------------------------------------------
 class TBackgroundTerminal;
 //---------------------------------------------------------------------------
-class TParallelUploadQueueItem : public TLocatedQueueItem
+class TParallelTransferQueueItem : public TLocatedQueueItem
 {
 public:
-  __fastcall TParallelUploadQueueItem(const TUploadQueueItem * ParentItem, TParallelOperation * ParallelOperation);
+  __fastcall TParallelTransferQueueItem(const TLocatedQueueItem * ParentItem, TParallelOperation * ParallelOperation);
 
 protected:
   virtual void __fastcall DoExecute(TTerminal * Terminal);
@@ -1732,18 +1732,6 @@ TQueueItem * __fastcall TQueueItem::CreateParallelOperation()
   return NULL;
 }
 //---------------------------------------------------------------------------
-void __fastcall TQueueItem::InitParallelOperationInfo(TQueueItem * Item)
-{
-  // deliberately not copying the ModifiedLocal and ModifiedRemote, not to trigger panel refresh, when sub-item completes
-  FInfo->Operation = Item->FInfo->Operation;
-  FInfo->Side = Item->FInfo->Side;
-  FInfo->Source = Item->FInfo->Source;
-  FInfo->Destination = Item->FInfo->Destination;
-  FInfo->SingleFile = DebugAlwaysFalse(Item->FInfo->SingleFile);
-  FInfo->Primary = false;
-  FInfo->GroupToken = Item->FInfo->GroupToken;
-}
-//---------------------------------------------------------------------------
 // TQueueItemProxy
 //---------------------------------------------------------------------------
 __fastcall TQueueItemProxy::TQueueItemProxy(TTerminalQueue * Queue,
@@ -2018,7 +2006,7 @@ void __fastcall TLocatedQueueItem::DoExecute(TTerminal * Terminal)
 __fastcall TTransferQueueItem::TTransferQueueItem(TTerminal * Terminal,
   TStrings * FilesToCopy, const UnicodeString & TargetDir,
   const TCopyParamType * CopyParam, int Params, TOperationSide Side,
-  bool SingleFile) :
+  bool SingleFile, bool Parallel) :
   TLocatedQueueItem(Terminal), FFilesToCopy(NULL), FCopyParam(NULL)
 {
   FInfo->Operation = (Params & cpDelete ? foMove : foCopy);
@@ -2040,6 +2028,10 @@ __fastcall TTransferQueueItem::TTransferQueueItem(TTerminal * Terminal,
   FCopyParam = new TCopyParamType(*CopyParam);
 
   FParams = Params;
+
+  FParallel = Parallel;
+  FLastParallelOperationAdded = GetTickCount();
+
 }
 //---------------------------------------------------------------------------
 __fastcall TTransferQueueItem::~TTransferQueueItem()
@@ -2057,62 +2049,16 @@ unsigned long __fastcall TTransferQueueItem::DefaultCPSLimit()
   return FCopyParam->CPSLimit;
 }
 //---------------------------------------------------------------------------
-// TUploadQueueItem
-//---------------------------------------------------------------------------
-__fastcall TUploadQueueItem::TUploadQueueItem(TTerminal * Terminal,
-  TStrings * FilesToCopy, const UnicodeString & TargetDir,
-  const TCopyParamType * CopyParam, int Params, bool SingleFile, bool Parallel) :
-  TTransferQueueItem(Terminal, FilesToCopy, TargetDir, CopyParam, Params, osLocal, SingleFile)
-{
-  if (FilesToCopy->Count > 1)
-  {
-    if (FLAGSET(Params, cpTemporary))
-    {
-      FInfo->Source = L"";
-      FInfo->ModifiedLocal = L"";
-    }
-    else
-    {
-      ExtractCommonPath(FilesToCopy, FInfo->Source);
-      // this way the trailing backslash is preserved for root directories like D:\\
-      FInfo->Source = ExtractFileDir(IncludeTrailingBackslash(FInfo->Source));
-      FInfo->ModifiedLocal = FLAGCLEAR(Params, cpDelete) ? UnicodeString() :
-        IncludeTrailingBackslash(FInfo->Source);
-    }
-  }
-  else
-  {
-    if (FLAGSET(Params, cpTemporary))
-    {
-      FInfo->Source = ExtractFileName(FilesToCopy->Strings[0]);
-      FInfo->ModifiedLocal = L"";
-    }
-    else
-    {
-      DebugAssert(FilesToCopy->Count > 0);
-      FInfo->Source = FilesToCopy->Strings[0];
-      FInfo->ModifiedLocal = FLAGCLEAR(Params, cpDelete) ? UnicodeString() :
-        IncludeTrailingBackslash(ExtractFilePath(FInfo->Source));
-    }
-  }
-
-  FInfo->Destination =
-    UnixIncludeTrailingBackslash(TargetDir) + CopyParam->FileMask;
-  FInfo->ModifiedRemote = UnixIncludeTrailingBackslash(TargetDir);
-  FParallel = Parallel;
-  FLastParallelOperationAdded = GetTickCount();
-}
-//---------------------------------------------------------------------------
-void __fastcall TUploadQueueItem::DoExecute(TTerminal * Terminal)
+void __fastcall TTransferQueueItem::DoExecute(TTerminal * Terminal)
 {
-  TTransferQueueItem::DoExecute(Terminal);
+  TLocatedQueueItem::DoExecute(Terminal);
 
   DebugAssert(Terminal != NULL);
-  TParallelOperation ParallelOperation(osLocal);
+  TParallelOperation ParallelOperation(FInfo->Side);
   FParallelOperation = &ParallelOperation;
   try
   {
-    Terminal->CopyToRemote(FFilesToCopy, FTargetDir, FCopyParam, FParams, &ParallelOperation);
+    DoTransferExecute(Terminal, FParallelOperation);
   }
   __finally
   {
@@ -2120,9 +2066,9 @@ void __fastcall TUploadQueueItem::DoExecute(TTerminal * Terminal)
   }
 }
 //---------------------------------------------------------------------------
-void __fastcall TUploadQueueItem::ProgressUpdated()
+void __fastcall TTransferQueueItem::ProgressUpdated()
 {
-  TTransferQueueItem::ProgressUpdated();
+  TLocatedQueueItem::ProgressUpdated();
 
   if (FParallel)
   {
@@ -2163,25 +2109,81 @@ void __fastcall TUploadQueueItem::ProgressUpdated()
   }
 }
 //---------------------------------------------------------------------------
-TQueueItem * __fastcall TUploadQueueItem::CreateParallelOperation()
+TQueueItem * __fastcall TTransferQueueItem::CreateParallelOperation()
 {
   DebugAssert(FParallelOperation != NULL);
 
   FParallelOperation->AddClient();
-  return new TParallelUploadQueueItem(this, FParallelOperation);
+  return new TParallelTransferQueueItem(this, FParallelOperation);
 }
 //---------------------------------------------------------------------------
-// TDownloadQueueItem
+// TUploadQueueItem
 //---------------------------------------------------------------------------
-__fastcall TParallelUploadQueueItem::TParallelUploadQueueItem(
-    const TUploadQueueItem * ParentItem, TParallelOperation * ParallelOperation) :
+__fastcall TUploadQueueItem::TUploadQueueItem(TTerminal * Terminal,
+  TStrings * FilesToCopy, const UnicodeString & TargetDir,
+  const TCopyParamType * CopyParam, int Params, bool SingleFile, bool Parallel) :
+  TTransferQueueItem(Terminal, FilesToCopy, TargetDir, CopyParam, Params, osLocal, SingleFile, Parallel)
+{
+  if (FilesToCopy->Count > 1)
+  {
+    if (FLAGSET(Params, cpTemporary))
+    {
+      FInfo->Source = L"";
+      FInfo->ModifiedLocal = L"";
+    }
+    else
+    {
+      ExtractCommonPath(FilesToCopy, FInfo->Source);
+      // this way the trailing backslash is preserved for root directories like D:\\
+      FInfo->Source = ExtractFileDir(IncludeTrailingBackslash(FInfo->Source));
+      FInfo->ModifiedLocal = FLAGCLEAR(Params, cpDelete) ? UnicodeString() :
+        IncludeTrailingBackslash(FInfo->Source);
+    }
+  }
+  else
+  {
+    if (FLAGSET(Params, cpTemporary))
+    {
+      FInfo->Source = ExtractFileName(FilesToCopy->Strings[0]);
+      FInfo->ModifiedLocal = L"";
+    }
+    else
+    {
+      DebugAssert(FilesToCopy->Count > 0);
+      FInfo->Source = FilesToCopy->Strings[0];
+      FInfo->ModifiedLocal = FLAGCLEAR(Params, cpDelete) ? UnicodeString() :
+        IncludeTrailingBackslash(ExtractFilePath(FInfo->Source));
+    }
+  }
+
+  FInfo->Destination =
+    UnixIncludeTrailingBackslash(TargetDir) + CopyParam->FileMask;
+  FInfo->ModifiedRemote = UnixIncludeTrailingBackslash(TargetDir);
+}
+//---------------------------------------------------------------------------
+void __fastcall TUploadQueueItem::DoTransferExecute(TTerminal * Terminal, TParallelOperation * ParallelOperation)
+{
+  Terminal->CopyToRemote(FFilesToCopy, FTargetDir, FCopyParam, FParams, ParallelOperation);
+}
+//---------------------------------------------------------------------------
+// TParallelTransferQueueItem
+//---------------------------------------------------------------------------
+__fastcall TParallelTransferQueueItem::TParallelTransferQueueItem(
+    const TLocatedQueueItem * ParentItem, TParallelOperation * ParallelOperation) :
   TLocatedQueueItem(*ParentItem),
   FParallelOperation(ParallelOperation)
 {
-  InitParallelOperationInfo(ParentItem);
+  // deliberately not copying the ModifiedLocal and ModifiedRemote, not to trigger panel refresh, when sub-item completes
+  FInfo->Operation = ParentItem->FInfo->Operation;
+  FInfo->Side = ParentItem->FInfo->Side;
+  FInfo->Source = ParentItem->FInfo->Source;
+  FInfo->Destination = ParentItem->FInfo->Destination;
+  FInfo->SingleFile = DebugAlwaysFalse(ParentItem->FInfo->SingleFile);
+  FInfo->Primary = false;
+  FInfo->GroupToken = ParentItem->FInfo->GroupToken;
 }
 //---------------------------------------------------------------------------
-void __fastcall TParallelUploadQueueItem::DoExecute(TTerminal * Terminal)
+void __fastcall TParallelTransferQueueItem::DoExecute(TTerminal * Terminal)
 {
   TLocatedQueueItem::DoExecute(Terminal);
 
@@ -2192,7 +2194,7 @@ void __fastcall TParallelUploadQueueItem::DoExecute(TTerminal * Terminal)
   OperationProgress.Start(
     // CPS limit inherited from parent OperationProgress.
     // Count not known and won't be needed as we will always have TotalSize as  we always transfer a single file at a time.
-    Operation, osLocal, -1, Temp, FParallelOperation->TargetDir, 0);
+    Operation, FParallelOperation->Side, -1, Temp, FParallelOperation->TargetDir, 0);
 
   try
   {
@@ -2222,8 +2224,8 @@ void __fastcall TParallelUploadQueueItem::DoExecute(TTerminal * Terminal)
 //---------------------------------------------------------------------------
 __fastcall TDownloadQueueItem::TDownloadQueueItem(TTerminal * Terminal,
   TStrings * FilesToCopy, const UnicodeString & TargetDir,
-  const TCopyParamType * CopyParam, int Params, bool SingleFile) :
-  TTransferQueueItem(Terminal, FilesToCopy, TargetDir, CopyParam, Params, osRemote, SingleFile)
+  const TCopyParamType * CopyParam, int Params, bool SingleFile, bool Parallel) :
+  TTransferQueueItem(Terminal, FilesToCopy, TargetDir, CopyParam, Params, osRemote, SingleFile, Parallel)
 {
   if (FilesToCopy->Count > 1)
   {
@@ -2265,12 +2267,9 @@ __fastcall TDownloadQueueItem::TDownloadQueueItem(TTerminal * Terminal,
   FInfo->ModifiedLocal = IncludeTrailingBackslash(TargetDir);
 }
 //---------------------------------------------------------------------------
-void __fastcall TDownloadQueueItem::DoExecute(TTerminal * Terminal)
+void __fastcall TDownloadQueueItem::DoTransferExecute(TTerminal * Terminal, TParallelOperation * ParallelOperation)
 {
-  TTransferQueueItem::DoExecute(Terminal);
-
-  DebugAssert(Terminal != NULL);
-  Terminal->CopyToLocal(FFilesToCopy, FTargetDir, FCopyParam, FParams);
+  Terminal->CopyToLocal(FFilesToCopy, FTargetDir, FCopyParam, FParams, ParallelOperation);
 }
 //---------------------------------------------------------------------------
 // TTerminalThread

source/core/Queue.h

@@ -66,8 +66,8 @@ class TTerminalQueue : public TSignalThread
 {
 friend class TQueueItem;
 friend class TQueueItemProxy;
-friend class TUploadQueueItem; //???
-friend class TParallelUploadQueueItem; //???
+friend class TTransferQueueItem;
+friend class TParallelTransferQueueItem;
 
 public:
   __fastcall TTerminalQueue(TTerminal * Terminal, TConfiguration * Configuration);
@@ -157,6 +157,7 @@ class TQueueItem
 {
 friend class TTerminalQueue;
 friend class TTerminalItem;
+friend class TParallelTransferQueueItem;
 
 public:
   enum TStatus {
@@ -206,7 +207,6 @@ protected:
   virtual UnicodeString __fastcall StartupDirectory() const = 0;
   virtual void __fastcall ProgressUpdated();
   virtual TQueueItem * __fastcall CreateParallelOperation();
-  void __fastcall InitParallelOperationInfo(TQueueItem * Item);
   bool __fastcall Complete();
 };
 //---------------------------------------------------------------------------
@@ -309,7 +309,7 @@ public:
   __fastcall TTransferQueueItem(TTerminal * Terminal,
     TStrings * FilesToCopy, const UnicodeString & TargetDir,
     const TCopyParamType * CopyParam, int Params, TOperationSide Side,
-    bool SingleFile);
+    bool SingleFile, bool Parallel);
   virtual __fastcall ~TTransferQueueItem();
 
 protected:
@@ -317,8 +317,15 @@ protected:
   UnicodeString FTargetDir;
   TCopyParamType * FCopyParam;
   int FParams;
+  bool FParallel;
+  DWORD FLastParallelOperationAdded;
+  TParallelOperation * FParallelOperation;
 
   virtual unsigned long __fastcall DefaultCPSLimit();
+  virtual void __fastcall DoExecute(TTerminal * Terminal);
+  virtual void __fastcall DoTransferExecute(TTerminal * Terminal, TParallelOperation * ParallelOperation) = 0;
+  virtual void __fastcall ProgressUpdated();
+  virtual TQueueItem * __fastcall CreateParallelOperation();
 };
 //---------------------------------------------------------------------------
 class TUploadQueueItem : public TTransferQueueItem
@@ -329,13 +336,7 @@ public:
     const TCopyParamType * CopyParam, int Params, bool SingleFile, bool Parallel);
 
 protected:
-  bool FParallel;
-  DWORD FLastParallelOperationAdded;
-  TParallelOperation * FParallelOperation;
-
-  virtual void __fastcall DoExecute(TTerminal * Terminal);
-  virtual void __fastcall ProgressUpdated();
-  virtual TQueueItem * __fastcall CreateParallelOperation();
+  virtual void __fastcall DoTransferExecute(TTerminal * Terminal, TParallelOperation * ParallelOperation);
 };
 //---------------------------------------------------------------------------
 class TDownloadQueueItem : public TTransferQueueItem
@@ -343,10 +344,10 @@ class TDownloadQueueItem : public TTransferQueueItem
 public:
   __fastcall TDownloadQueueItem(TTerminal * Terminal,
     TStrings * FilesToCopy, const UnicodeString & TargetDir,
-    const TCopyParamType * CopyParam, int Params, bool SingleFile);
+    const TCopyParamType * CopyParam, int Params, bool SingleFile, bool Parallel);
 
 protected:
-  virtual void __fastcall DoExecute(TTerminal * Terminal);
+  virtual void __fastcall DoTransferExecute(TTerminal * Terminal, TParallelOperation * ParallelOperation);
 };
 //---------------------------------------------------------------------------
 class TUserAction;

source/core/Script.cpp

@@ -1432,7 +1432,7 @@ void __fastcall TScript::GetProc(TScriptProcParams * Parameters)
     CopyParamParams(CopyParam, Parameters);
     CheckParams(Parameters);
 
-    FTerminal->CopyToLocal(FileList, TargetDirectory, &CopyParam, Params);
+    FTerminal->CopyToLocal(FileList, TargetDirectory, &CopyParam, Params, NULL);
   }
   __finally
   {

source/core/Terminal.cpp

@@ -699,10 +699,11 @@ TCollectedFileList::TCollectedFileList()
 {
 }
 //---------------------------------------------------------------------------
-int TCollectedFileList::Add(const UnicodeString & FileName, bool Dir)
+int TCollectedFileList::Add(const UnicodeString & FileName, TObject * Object, bool Dir)
 {
   TFileData Data;
   Data.FileName = FileName;
+  Data.Object = Object;
   Data.Dir = Dir;
   Data.Recursed = true;
   FList.push_back(Data);
@@ -729,6 +730,11 @@ UnicodeString TCollectedFileList::GetFileName(int Index) const
   return FList[Index].FileName;
 }
 //---------------------------------------------------------------------------
+TObject * TCollectedFileList::GetObject(int Index) const
+{
+  return FList[Index].Object;
+}
+//---------------------------------------------------------------------------
 bool TCollectedFileList::IsDir(int Index) const
 {
   return FList[Index].Dir;
@@ -893,7 +899,7 @@ bool TParallelOperation::CheckEnd(TCollectedFileList * Files)
   return Result;
 }
 //---------------------------------------------------------------------------
-int TParallelOperation::GetNext(TTerminal * Terminal, UnicodeString & FileName, UnicodeString & TargetDir, bool & Dir, bool & Recursed)
+int TParallelOperation::GetNext(TTerminal * Terminal, UnicodeString & FileName, TObject *& Object, UnicodeString & TargetDir, bool & Dir, bool & Recursed)
 {
   int Result = 1;
   TCollectedFileList * Files;
@@ -921,6 +927,7 @@ int TParallelOperation::GetNext(TTerminal * Terminal, UnicodeString & FileName,
     UnicodeString RootPath = FFileList->Strings[0];
 
     FileName = Files->GetFileName(FIndex);
+    Object = Files->GetObject(FIndex);
     Dir = Files->IsDir(FIndex);
     Recursed = Files->IsRecursed(FIndex);
     UnicodeString DirPath;
@@ -4128,6 +4135,26 @@ bool __fastcall TTerminal::LoadFilesProperties(TStrings * FileList)
   return Result;
 }
 //---------------------------------------------------------------------------
+void __fastcall TTerminal::DoCalculateFileSize(UnicodeString FileName,
+  const TRemoteFile * File, void * Param)
+{
+  TCalculateSizeParams * AParams = static_cast<TCalculateSizeParams*>(Param);
+
+  if (AParams->Stats->FoundFiles != NULL)
+  {
+    UnicodeString DirPath = UnixExtractFilePath(UnixExcludeTrailingBackslash(File->FullFileName));
+    if ((DirPath != AParams->LastDirPath) ||
+        (AParams->Files == NULL))
+    {
+      AParams->Files = new TCollectedFileList();
+      AParams->LastDirPath = DirPath;
+      AParams->Stats->FoundFiles->AddObject(AParams->LastDirPath, AParams->Files);
+    }
+  }
+
+  CalculateFileSize(FileName, File, Param);
+}
+//---------------------------------------------------------------------------
 void __fastcall TTerminal::CalculateFileSize(UnicodeString FileName,
   const TRemoteFile * File, /*TCalculateSizeParams*/ void * Param)
 {
@@ -4163,6 +4190,13 @@ void __fastcall TTerminal::CalculateFileSize(UnicodeString FileName,
 
   if (AllowTransfer)
   {
+    int CollectionIndex = -1;
+    if (AParams->Files != NULL)
+    {
+      UnicodeString FullFileName = UnixExcludeTrailingBackslash(File->FullFileName);
+      CollectionIndex = AParams->Files->Add(FullFileName, File->Duplicate(), File->IsDirectory);
+    }
+
     if (File->IsDirectory)
     {
       if (CanRecurseToDirectory(File))
@@ -4175,41 +4209,43 @@ void __fastcall TTerminal::CalculateFileSize(UnicodeString FileName,
         {
           LogEvent(FORMAT(L"Getting size of directory \"%s\"", (FileName)));
           // pass in full path so we get it back in file list for AllowTransfer() exclusion
-          DoCalculateDirectorySize(File->FullFileName, File, AParams);
+          if (!DoCalculateDirectorySize(File->FullFileName, File, AParams))
+          {
+            if (CollectionIndex >= 0)
+            {
+              AParams->Files->DidNotRecurse(CollectionIndex);
+            }
+          }
         }
       }
 
-      if (AParams->Stats != NULL)
-      {
-        AParams->Stats->Directories++;
-      }
+      AParams->Stats->Directories++;
     }
     else
     {
       AParams->Size += File->Size;
 
-      if (AParams->Stats != NULL)
-      {
-        AParams->Stats->Files++;
-      }
+      AParams->Stats->Files++;
     }
 
-    if ((AParams->Stats != NULL) && File->IsSymLink)
+    if (File->IsSymLink)
     {
       AParams->Stats->SymLinks++;
     }
   }
 }
 //---------------------------------------------------------------------------
-void __fastcall TTerminal::DoCalculateDirectorySize(const UnicodeString FileName,
+bool __fastcall TTerminal::DoCalculateDirectorySize(const UnicodeString FileName,
   const TRemoteFile * /*File*/, TCalculateSizeParams * Params)
 {
+  bool Result = false;
   TRetryOperationLoop RetryLoop(this);
   do
   {
     try
     {
       ProcessDirectory(FileName, CalculateFileSize, Params);
+      Result = true;
     }
     catch(Exception & E)
     {
@@ -4221,11 +4257,12 @@ void __fastcall TTerminal::DoCalculateDirectorySize(const UnicodeString FileName
     }
   }
   while (RetryLoop.Retry());
+  return Result;
 }
 //---------------------------------------------------------------------------
 bool __fastcall TTerminal::CalculateFilesSize(TStrings * FileList,
   __int64 & Size, int Params, const TCopyParamType * CopyParam,
-  bool AllowDirs, TCalculateSizeStats * Stats)
+  bool AllowDirs, TCalculateSizeStats & Stats)
 {
   // With FTP protocol, we may use DSIZ command from
   // draft-peterson-streamlined-ftp-command-extensions-10
@@ -4238,10 +4275,11 @@ bool __fastcall TTerminal::CalculateFilesSize(TStrings * FileList,
   Param.Size = 0;
   Param.Params = Params;
   Param.CopyParam = CopyParam;
-  Param.Stats = Stats;
+  Param.Stats = &Stats;
   Param.AllowDirs = AllowDirs;
   Param.Result = true;
-  ProcessFiles(FileList, foCalculateSize, CalculateFileSize, &Param);
+  Param.Files = NULL;
+  ProcessFiles(FileList, foCalculateSize, DoCalculateFileSize, &Param);
   Size = Param.Size;
   return Param.Result;
 }
@@ -5070,7 +5108,7 @@ void __fastcall TTerminal::CalculateLocalFileSize(const UnicodeString FileName,
         if (AParams->Files != NULL)
         {
           UnicodeString FullFileName = ::ExpandFileName(FileName);
-          CollectionIndex = AParams->Files->Add(FullFileName, Dir);
+          CollectionIndex = AParams->Files->Add(FullFileName, NULL, Dir);
         }
 
         if (!Dir)
@@ -5853,7 +5891,7 @@ void __fastcall TTerminal::SynchronizeApply(TSynchronizeChecklist * Checklist,
         else
         {
           if ((DownloadList->Count > 0) &&
-              !CopyToLocal(DownloadList, Data.LocalDirectory, &SyncCopyParam, CopyParams))
+              !CopyToLocal(DownloadList, Data.LocalDirectory, &SyncCopyParam, CopyParams, NULL))
           {
             Abort();
           }
@@ -6207,15 +6245,16 @@ bool __fastcall TTerminal::GetStoredCredentialsTried()
 int __fastcall TTerminal::CopyToParallel(TParallelOperation * ParallelOperation, TFileOperationProgressType * OperationProgress)
 {
   UnicodeString FileName;
+  TObject * Object;
   UnicodeString TargetDir;
   bool Dir;
   bool Recursed;
 
-  int Result = ParallelOperation->GetNext(this, FileName, TargetDir, Dir, Recursed);
+  int Result = ParallelOperation->GetNext(this, FileName, Object, TargetDir, Dir, Recursed);
   if (Result > 0)
   {
     std::unique_ptr<TStrings> FilesToCopy(new TStringList());
-    FilesToCopy->Add(FileName);
+    FilesToCopy->AddObject(FileName, Object);
 
     if (ParallelOperation->Side == osLocal)
     {
@@ -6234,20 +6273,23 @@ int __fastcall TTerminal::CopyToParallel(TParallelOperation * ParallelOperation,
     int Prev = OperationProgress->FilesFinishedSuccessfully;
     try
     {
+      FOperationProgress = OperationProgress;
       if (ParallelOperation->Side == osLocal)
       {
         FFileSystem->CopyToRemote(
           FilesToCopy.get(), TargetDir, ParallelOperation->CopyParam, Params, OperationProgress, OnceDoneOperation);
       }
-      else
+      else if (DebugAlwaysTrue(ParallelOperation->Side == osRemote))
       {
-        DebugFail();
+        FFileSystem->CopyToLocal(
+          FilesToCopy.get(), TargetDir, ParallelOperation->CopyParam, Params, OperationProgress, OnceDoneOperation);
       }
     }
     __finally
     {
       bool Success = (Prev < OperationProgress->FilesFinishedSuccessfully);
       ParallelOperation->Done(FileName, Dir, Success);
+      FOperationProgress = NULL;
     }
   }
 
@@ -6406,7 +6448,8 @@ bool __fastcall TTerminal::CopyToRemote(TStrings * FilesToCopy,
 }
 //---------------------------------------------------------------------------
 bool __fastcall TTerminal::CopyToLocal(TStrings * FilesToCopy,
-  const UnicodeString TargetDir, const TCopyParamType * CopyParam, int Params)
+  const UnicodeString TargetDir, const TCopyParamType * CopyParam, int Params,
+  TParallelOperation * ParallelOperation)
 {
   DebugAssert(FFileSystem);
 
@@ -6423,14 +6466,23 @@ bool __fastcall TTerminal::CopyToLocal(TStrings * FilesToCopy,
     bool TotalSizeKnown = false;
     TFileOperationProgressType OperationProgress(&DoProgress, &DoFinished);
 
+    std::unique_ptr<TStringList> Files;
+    if (CanParallel(CopyParam, Params, ParallelOperation))
+    {
+      Files.reset(new TStringList());
+      Files->OwnsObjects = true;
+    }
+
     ExceptionOnFail = true;
     try
     {
       // dirty trick: when moving, do not pass copy param to avoid exclude mask
+      TCalculateSizeStats Stats;
+      Stats.FoundFiles = Files.get();
       if (CalculateFilesSize(
            FilesToCopy, TotalSize, csIgnoreErrors,
            (FLAGCLEAR(Params, cpDelete) ? CopyParam : NULL),
-           CopyParam->CalculateSize, NULL))
+           CopyParam->CalculateSize, Stats))
       {
         TotalSizeKnown = true;
       }
@@ -6465,15 +6517,26 @@ bool __fastcall TTerminal::CopyToLocal(TStrings * FilesToCopy,
       {
         try
         {
+          bool Parallel = TotalSizeKnown && (Files.get() != NULL);
           if (Log->Logging)
           {
-            LogEvent(FORMAT(L"Copying %d files/directories to local directory "
-              "\"%s\"", (FilesToCopy->Count, TargetDir)));
+            LogEvent(FORMAT(
+              L"Copying %d files/directories to local directory \"%s\"%s",
+              (FilesToCopy->Count, TargetDir, (Parallel ? L" in parallel" : L""))));
             LogEvent(CopyParam->LogStr);
           }
 
-          FFileSystem->CopyToLocal(FilesToCopy, TargetDir, CopyParam, Params,
-            &OperationProgress, OnceDoneOperation);
+          if (Parallel)
+          {
+            // OnceDoneOperation is not supported
+            ParallelOperation->Init(Files.release(), TargetDir, CopyParam, Params, &OperationProgress);
+            CopyParallel(ParallelOperation, &OperationProgress);
+          }
+          else
+          {
+            FFileSystem->CopyToLocal(FilesToCopy, TargetDir, CopyParam, Params,
+              &OperationProgress, OnceDoneOperation);
+          }
         }
         __finally
         {

source/core/Terminal.h

@@ -295,7 +295,9 @@ protected:
   bool __fastcall HandleException(Exception * E);
   void __fastcall CalculateFileSize(UnicodeString FileName,
     const TRemoteFile * File, /*TCalculateSizeParams*/ void * Size);
-  void __fastcall DoCalculateDirectorySize(const UnicodeString FileName,
+  void __fastcall DoCalculateFileSize(UnicodeString FileName,
+    const TRemoteFile * File, void * Param);
+  bool __fastcall DoCalculateDirectorySize(const UnicodeString FileName,
     const TRemoteFile * File, TCalculateSizeParams * Params);
   void __fastcall CalculateLocalFileSize(const UnicodeString FileName,
     const TSearchRec Rec, /*__int64*/ void * Size);
@@ -430,7 +432,7 @@ public:
   bool __fastcall FileExists(const UnicodeString FileName, TRemoteFile ** File = NULL);
   void __fastcall ReadSymlink(TRemoteFile * SymlinkFile, TRemoteFile *& File);
   bool __fastcall CopyToLocal(TStrings * FilesToCopy,
-    const UnicodeString TargetDir, const TCopyParamType * CopyParam, int Params);
+    const UnicodeString TargetDir, const TCopyParamType * CopyParam, int Params, TParallelOperation * ParallelOperation);
   bool __fastcall CopyToRemote(TStrings * FilesToCopy,
     const UnicodeString TargetDir, const TCopyParamType * CopyParam, int Params, TParallelOperation * ParallelOperation);
   int __fastcall CopyToParallel(TParallelOperation * ParallelOperation, TFileOperationProgressType * OperationProgress);
@@ -470,7 +472,7 @@ public:
     const UnicodeString FileMask);
   bool __fastcall CalculateFilesSize(TStrings * FileList, __int64 & Size,
     int Params, const TCopyParamType * CopyParam, bool AllowDirs,
-    TCalculateSizeStats * Stats);
+    TCalculateSizeStats & Stats);
   void __fastcall CalculateFilesChecksum(const UnicodeString & Alg, TStrings * FileList,
     TStrings * Checksums, TCalculatedChecksumEvent OnCalculatedChecksum);
   void __fastcall ClearCaches();
@@ -618,6 +620,7 @@ struct TCalculateSizeStats
   int Files;
   int Directories;
   int SymLinks;
+  TStrings * FoundFiles;
 };
 //---------------------------------------------------------------------------
 struct TCalculateSizeParams
@@ -628,6 +631,7 @@ struct TCalculateSizeParams
   TCalculateSizeStats * Stats;
   bool AllowDirs;
   TCollectedFileList * Files;
+  UnicodeString LastDirPath;
   bool Result;
 };
 //---------------------------------------------------------------------------
@@ -762,12 +766,13 @@ class TCollectedFileList : public TObject
 {
 public:
   TCollectedFileList();
-  int Add(const UnicodeString & FileName, bool Dir);
+  int Add(const UnicodeString & FileName, TObject * Object, bool Dir);
   void DidNotRecurse(int Index);
   void Delete(int Index);
 
   int Count() const;
   UnicodeString GetFileName(int Index) const;
+  TObject * GetObject(int Index) const;
   bool IsDir(int Index) const;
   bool IsRecursed(int Index) const;
 
@@ -775,6 +780,7 @@ private:
   struct TFileData
   {
     UnicodeString FileName;
+    TObject * Object;
     bool Dir;
     bool Recursed;
   };
@@ -796,7 +802,9 @@ public:
   bool ShouldAddClient();
   void AddClient();
   void RemoveClient();
-  int GetNext(TTerminal * Terminal, UnicodeString & FileName, UnicodeString & TargetDir, bool & Dir, bool & Recursed);
+  int GetNext(
+    TTerminal * Terminal, UnicodeString & FileName, TObject *& Object, UnicodeString & TargetDir,
+    bool & Dir, bool & Recursed);
   void Done(const UnicodeString & FileName, bool Dir, bool Success);
 
   __property TOperationSide Side = { read = FSide };

source/forms/CustomScpExplorer.cpp

@@ -1165,7 +1165,7 @@ void __fastcall TCustomScpExplorerForm::AddQueueItem(
   else
   {
     QueueItem = new TDownloadQueueItem(Terminal, FileList, TargetDirectory,
-      &CopyParam, Params, SingleFile);
+      &CopyParam, Params, SingleFile, CopyParam.QueueParallel);
   }
   AddQueueItem(Queue, QueueItem, Terminal);
 }
@@ -2371,8 +2371,8 @@ bool __fastcall TCustomScpExplorerForm::ExecuteFileOperation(TFileOperation Oper
 
               try
               {
-                Terminal->CopyToLocal(FileList, TargetDirectory, &CopyParam,
-                  Params);
+                Terminal->CopyToLocal(
+                  FileList, TargetDirectory, &CopyParam, Params, NULL);
               }
               __finally
               {
@@ -3045,8 +3045,8 @@ void __fastcall TCustomScpExplorerForm::TemporarilyDownloadFiles(
     {
       // turn off confirmations, as for MDI editors we may possibly download
       // the same file over
-      Terminal->CopyToLocal(FileList, TempDir, &CopyParam,
-        cpNoConfirmation | cpTemporary);
+      Terminal->CopyToLocal(
+        FileList, TempDir, &CopyParam, cpNoConfirmation | cpTemporary, NULL);
 
       if (GetTargetNames)
       {
@@ -3946,7 +3946,7 @@ void __fastcall TCustomScpExplorerForm::CalculateSize(
   {
     try
     {
-      Terminal->CalculateFilesSize(FileList, Size, 0, NULL, true, &Stats);
+      Terminal->CalculateFilesSize(FileList, Size, 0, NULL, true, Stats);
     }
     catch(...)
     {
@@ -6891,7 +6891,7 @@ void __fastcall TCustomScpExplorerForm::DDDownload(TStrings * FilesToCopy,
   try
   {
     UpdateCopyParamCounters(*CopyParam);
-    Terminal->CopyToLocal(FilesToCopy, TargetDir, CopyParam, Params);
+    Terminal->CopyToLocal(FilesToCopy, TargetDir, CopyParam, Params, NULL);
     if (FLAGSET(Params, cpDelete) && (DropSourceControl == RemoteDriveView))
     {
       RemoteDriveView->UpdateDropSource();

source/windows/WinMain.cpp

@@ -163,7 +163,7 @@ void __fastcall Download(TTerminal * Terminal, const UnicodeString FileName,
 
       std::unique_ptr<TStrings> FileList(new TStringList());
       FileList->AddObject(FileName, File);
-      Terminal->CopyToLocal(FileList.get(), TargetDirectory, &CopyParam, 0);
+      Terminal->CopyToLocal(FileList.get(), TargetDirectory, &CopyParam, 0, NULL);
     }
 
     UnicodeString Directory = UnixExtractFilePath(FileName);