Bug 1631: Optimize loading of large directories

https://winscp.net/tracker/1631
(cherry picked from commit 27f61a0b10f5c1bb14379d3551a3165e19f848fe)

Source commit: cad4585dc8759f793fe2c5a313d2787605da3492

source/components/UnixDirView.cpp

@@ -323,6 +323,7 @@ void __fastcall TUnixDirView::LoadFiles()
     int VisibleFiles = 0;
     FHiddenCount = 0;
     FFilteredCount = 0;
+    DebugAssert(Items->Count == 0); // to make sure that Index matches Items->Count
     for (int Index = 0; Index < Terminal->Files->Count; Index++)
     {
       TRemoteFile *File = Terminal->Files->Files[Index];
@@ -347,11 +348,16 @@ void __fastcall TUnixDirView::LoadFiles()
 
         TListItem * Item = new TListItem(Items);
         Item->Data = File;
-        // Need to add before assigning to .Caption and .OverlayIndex,
-        // as the setters these call back to owning view.
-        // Assignment is redundant
-        Item = Items->AddItem(Item);
-        Item->Caption = File->FileName;
+        // Need to add before assigning to .Caption, as its setter call back to owning view.
+        // Item assignment is redundant.
+        // Index is optimization.
+        Item = Items->AddItem(Item, Index);
+        // Setting Caption is expensive and it's for display only.
+        // Captions of excessive items is delay loaded in GetDisplayInfo.
+        if (Index <= 10000)
+        {
+          Item->Caption = File->FileName;
+        }
         if (FFullLoad)
         {
           // this is out of date
@@ -367,7 +373,7 @@ void __fastcall TUnixDirView::LoadFiles()
       }
     }
 
-    if (OwnerData)
+    if (DebugAlwaysFalse(OwnerData))
     {
       Items->Count = VisibleFiles;
     }
@@ -381,6 +387,11 @@ void __fastcall TUnixDirView::GetDisplayInfo(TListItem * Item, tagLVITEMW &DispI
   {
 #ifndef DESIGN_ONLY
     TRemoteFile * File = ITEMFILE;
+    // delay loading caption
+    if (Item->Caption.IsEmpty())
+    {
+      Item->Caption = File->FileName;
+    }
     if (DispInfo.mask & LVIF_TEXT)
     {
       UnicodeString Value;

source/core/FtpFileSystem.cpp

@@ -1500,14 +1500,15 @@ void __fastcall TFTPFileSystem::ResetFileTransfer()
 //---------------------------------------------------------------------------
 void __fastcall TFTPFileSystem::ReadDirectoryProgress(__int64 Bytes)
 {
-  // with FTP we do not know exactly how many entries we have received,
-  // instead we know number of bytes received only.
-  // so we report approximation based on average size of entry.
-  int Progress = int(Bytes / 80);
-  if (Progress - FLastReadDirectoryProgress >= 10)
-  {
+  DWORD Ticks = GetTickCount();
+  if (Ticks - FLastReadDirectoryProgress >= 100)
+  {
+    FLastReadDirectoryProgress = Ticks;
+    // with FTP we do not know exactly how many entries we have received,
+    // instead we know number of bytes received only.
+    // so we report approximation based on average size of entry.
+    int Progress = int(Bytes / 80);
     bool Cancel = false;
-    FLastReadDirectoryProgress = Progress;
     FTerminal->DoReadDirectoryProgress(Progress, 0, Cancel);
     if (Cancel)
     {

source/core/FtpFileSystem.h

@@ -245,7 +245,7 @@ private:
   bool FMultiLineResponse;
   int FLastCode;
   int FLastCodeClass;
-  int FLastReadDirectoryProgress;
+  DWORD FLastReadDirectoryProgress;
   UnicodeString FTimeoutStatus;
   UnicodeString FDisconnectStatus;
   TStrings * FLastResponse;

source/forms/CustomScpExplorer.cpp

@@ -10920,10 +10920,10 @@ TListItem * __fastcall TCustomScpExplorerForm::SearchFile(const UnicodeString &
       switch (WinConfiguration->PanelSearch)
       {
         case isNameStartOnly:
-          Matches = ContainsTextSemiCaseSensitive(Item->Caption.SubString(1, Text.Length()), Text);
+          Matches = ContainsTextSemiCaseSensitive(ADirView->ItemFileName(Item).SubString(1, Text.Length()), Text);
           break;
         case isName:
-          Matches = ContainsTextSemiCaseSensitive(Item->Caption, Text);
+          Matches = ContainsTextSemiCaseSensitive(ADirView->ItemFileName(Item), Text);
           break;
         case isAll:
           int ColCount = (ADirView->ViewStyle == vsReport) ? ADirView->ColProperties->Count : 1;

source/packages/filemng/CustomDirView.pas

@@ -1398,9 +1398,9 @@ begin
     begin
       Item := Items[Index];
       Assert(Assigned(Item));
-      if (Item.Selected <> Select) and
+      if (GetItemSelectedByIndex(Index) <> Select) and
          ItemMatchesFilter(Item, Filter) then
-           Item.Selected := Select;
+           SetItemSelectedByIndex(Index, Select);
     end;
   finally
     Screen.Cursor := OldCursor;
@@ -1850,7 +1850,7 @@ begin
         // cannot use ItemFileName as for TUnixDirView the file object
         // is no longer valid
         FileName := Item.Caption;
-        if Item.Selected then
+        if GetItemSelectedByIndex(Index) then
           OldSelection.Add(FileName);
 
         if CacheIcons and (ItemImageIndex(Item, True) >= 0) then
@@ -1923,7 +1923,7 @@ begin
           ItemToFocus := Item;
 
         if OldSelection.Find(FileName, FoundIndex) then
-          Item.Selected := True;
+          SetItemSelectedByIndex(Index, True);
 
         if CacheIcons and (ItemImageIndex(Item, True) < 0) then
         begin
@@ -1993,6 +1993,7 @@ begin
     else
   begin
     FLoading := True;
+    FInsertingNewUnselectedItem := True;
     try
       FHasParentDir := False;
 
@@ -2019,6 +2020,7 @@ begin
       end;
     finally
       FLoading := False;
+      FInsertingNewUnselectedItem := False;
 
       try
         if FAbortLoading then

source/packages/my/NortonLikeListView.pas

@@ -52,6 +52,7 @@ type
   protected
     { Protected declarations }
     FClearingItems: Boolean;
+    FInsertingNewUnselectedItem: Boolean;
     FUpdatingSelection: Integer;
     FNextCharToIgnore: Word;
     FHeaderHandle: HWND;
@@ -76,6 +77,8 @@ type
     function CanEdit(Item: TListItem): Boolean; override;
     function GetPopupMenu: TPopupMenu; override;
     procedure ChangeScale(M, D: Integer); override;
+    procedure SetItemSelectedByIndex(Index: Integer; Select: Boolean);
+    function GetItemSelectedByIndex(Index: Integer): Boolean;
   public
     { Public declarations }
     constructor Create(AOwner: TComponent); override;
@@ -113,6 +116,7 @@ begin
   FFirstSelected := -1;
   FLastSelected := -1;
   FClearingItems := False;
+  FInsertingNewUnselectedItem := False;
   MultiSelect := True;
   FDontSelectItem := False;
   FDontUnSelectItem := False;
@@ -220,7 +224,7 @@ begin
   begin
     Index := Item.Index;
 
-    if Item.Selected then
+    if GetItemSelectedByIndex(Index) then
       ItemUnselected(Item, Index);
 
     if FManageSelection then
@@ -297,13 +301,13 @@ begin
   if Assigned(Item) and (Item.Selected or ((NortonLike <> nlOff) and (SelCount = 0))) then
   begin
     Index := Item.Index + 1;
-    while (Index < Items.Count) and Items[Index].Selected do Inc(Index);
-    if (Index >= Items.Count) or Items[Index].Selected then
+    while (Index < Items.Count) and GetItemSelectedByIndex(Index) do Inc(Index);
+    if (Index >= Items.Count) or GetItemSelectedByIndex(Index) then
     begin
       Index := Item.Index - 1;
-      while (Index >= 0) and Items[Index].Selected do Dec(Index);
+      while (Index >= 0) and GetItemSelectedByIndex(Index) do Dec(Index);
     end;
-    if (Index >= 0) and (Index < Items.Count) and (not Items[Index].Selected) then
+    if (Index >= 0) and (Index < Items.Count) and (not GetItemSelectedByIndex(Index)) then
       Result := Items[Index]
     else
       Result := nil;
@@ -679,22 +683,64 @@ begin
     ItemFocused := Item;
 end;
 
+// TListItem.Selected needs an index, which is expensively looked up.
+// If we know it already, avoid that loop up.
+procedure TCustomNortonLikeListView.SetItemSelectedByIndex(Index: Integer; Select: Boolean);
+var
+  State: Integer;
+begin
+  if Select then State := LVIS_SELECTED
+    else State := 0;
+  ListView_SetItemState(Handle, Index, State, LVIS_SELECTED);
+end;
+
+function TCustomNortonLikeListView.GetItemSelectedByIndex(Index: Integer): Boolean;
+begin
+  Result := (ListView_GetItemState(Handle, Index, LVIS_SELECTED) and LVIS_SELECTED) <> 0;
+end;
+
 procedure TCustomNortonLikeListView.SelectAll(Mode: TSelectMode; Exclude: TListItem);
 var
   Index: Integer;
   Item: TListItem;
+  NewState: Boolean;
 begin
   BeginSelectionUpdate;
   try
-    for Index := 0 to Items.Count - 1 do
+    // Setting/Querying selected state is expensive.
+    // This optimization is important for call from TCustomNortonLikeListView.WMLButtonUp in nlKeyboard mode.
+    if (Mode = smNone) and
+       // If there are too many, plain iteration is more effective then using GetNextItem
+       // (though that can be optimized too, by passing index in and out instead of an item pointer)
+       (FSelCount < Items.Count div 4) then
+    begin
+      Item := GetNextItem(nil, sdAll, [isSelected]);
+      while Assigned(Item) do
+      begin
+        if Item <> Exclude then
+          Item.Selected := False;
+        Item := GetNextItem(Item, sdAll, [isSelected]);
+      end;
+    end
+      else
     begin
-      Item := Items[Index];
-      if Item <> Exclude then
+      for Index := 0 to Items.Count - 1 do
       begin
-        case Mode of
-          smAll: Item.Selected := True;
-          smNone: Item.Selected := False;
-          smInvert: Item.Selected := not Item.Selected;
+        Item := Items[Index];
+        if Item <> Exclude then
+        begin
+          case Mode of
+            smAll: NewState := True;
+            smNone: NewState := False;
+            smInvert: NewState := not GetItemSelectedByIndex(Index);
+              else
+            begin
+              Assert(False);
+              NewState := False;
+            end;
+          end;
+
+          SetItemSelectedByIndex(Index, NewState);
         end;
       end;
     end;
@@ -862,7 +908,7 @@ begin
           else
         begin
           Index := First;
-          while (Index <= Last) and (not (Items[Index].Selected)) do
+          while (Index <= Last) and (not GetItemSelectedByIndex(Index)) do
           begin
             Inc(Index);
           end;
@@ -871,7 +917,7 @@ begin
           begin
             Result := nil;
 
-            if Assigned(StartItem) and StartItem.Selected then
+            if (Start >= 0) and GetItemSelectedByIndex(Start) then
             begin
               Assert((FLastSelected < 0) or (FLastSelected = Start));
               FLastSelected := Start;
@@ -880,7 +926,7 @@ begin
             else
           begin
             Result := Items[Index];
-            Assert(Result.Selected);
+            Assert(GetItemSelectedByIndex(Index));
 
             if not Assigned(StartItem) then
             begin
@@ -925,8 +971,12 @@ end;
 procedure TCustomNortonLikeListView.InsertItem(Item: TListItem);
 begin
   inherited;
-  if Item.Selected then
+
+  if (not FInsertingNewUnselectedItem) and // Optimization to avoid expensive Item.Selected
+     Item.Selected then
+  begin
     ItemSelected(Item, -1);
+  end;
 end;
 
 function TCustomNortonLikeListView.GetItemFromHItem(const Item: TLVItem): TListItem;