Sped up loading list with large databases 30000+ entries

git-svn-id: svn://svn.code.sf.net/p/ditto-cp/code/trunk@762 595ec19a-5cb4-439b-94a8-42fb3063c22c

Clip.cpp

@@ -10,6 +10,7 @@
 #include "sqlite\CppSQLite3.h"
 #include "shared/TextConvert.h"
 #include "zlib/zlib.h"
+#include "Misc.h"
 
 #include <Mmsystem.h>
 
@@ -167,8 +168,8 @@ CClip::CClip() :
 	m_bIsGroup(FALSE),
 	m_param1(0),
 	m_clipOrder(0),
-	m_stickyClipOrder(0),
-	m_stickyClipGroupOrder(0),
+	m_stickyClipOrder(INVALID_STICKY),
+	m_stickyClipGroupOrder(INVALID_STICKY),
 	m_clipGroupOrder(0),
 	m_globalShortCut(FALSE)
 {
@@ -755,7 +756,7 @@ bool CClip::AddToDataTable()
 
 void CClip::MoveUp()
 {
-	if(m_stickyClipOrder == 0)
+	if (m_stickyClipOrder == INVALID_STICKY)
 	{
 		CppSQLite3Query q = theApp.m_db.execQueryEx(_T("SELECT lID, clipOrder FROM Main Where clipOrder > %f ORDER BY clipOrder ASC LIMIT 1"), m_clipOrder);
 		if (q.eof() == false)
@@ -777,10 +778,7 @@ void CClip::MoveUp()
 			}
 		}
 	}
-	else  
-
-
-
+	else 
 	{
 		CppSQLite3Query q = theApp.m_db.execQueryEx(_T("SELECT lID, stickyClipOrder FROM Main Where stickyClipOrder > %f ORDER BY clipOrder ASC LIMIT 1"), m_stickyClipOrder);
 		if (q.eof() == false)
@@ -832,11 +830,11 @@ void CClip::RemoveStickySetting(int parentId)
 {
 	if (parentId < 0)
 	{
-		m_stickyClipOrder = 0;
+		m_stickyClipOrder = INVALID_STICKY;
 	}
 	else
 	{
-		m_stickyClipGroupOrder = 0;
+		m_stickyClipGroupOrder = INVALID_STICKY;
 	}
 }
 
@@ -888,7 +886,7 @@ double CClip::GetNewLastSticky(int parentId, int clipId)
 	{
 		if (parentId < 0)
 		{
-			CppSQLite3Query q = theApp.m_db.execQuery(_T("SELECT stickyClipOrder, mText FROM Main ORDER BY stickyClipOrder LIMIT 1"));
+			CppSQLite3Query q = theApp.m_db.execQuery(_T("SELECT stickyClipOrder, mText FROM Main WHERE stickyClipOrder <> -10000000 ORDER BY stickyClipOrder LIMIT 1"));
 			if (q.eof() == false)
 			{
 				existingMaxOrder = q.getFloatField(_T("stickyClipOrder"));

DatabaseUtilities.cpp

@@ -389,6 +389,40 @@ BOOL ValidDB(CString csPath, BOOL bUpgrade)
 			db.execDML(_T("ALTER TABLE Main ADD stickyClipOrder REAL"));
 			db.execDML(_T("ALTER TABLE Main ADD stickyClipGroupOrder REAL"));
 
+			e.errorCode();
+		}
+
+		try
+		{			
+			CppSQLite3Query q = db.execQuery(_T("PRAGMA index_info(Main_NoGroup);"));
+			int count = 0;
+			while (q.eof() == false)
+			{
+				count++;
+				q.nextRow();
+			}
+
+			if(count == 0)
+			{
+				if (didBackup == FALSE)
+					didBackup = BackupDB(csPath, backupFilePrefix);				
+
+				db.execDML(_T("Update Main set stickyClipOrder = -(2147483647) where stickyClipOrder IS NULL;"));
+				db.execDML(_T("Update Main set stickyClipGroupOrder = -(2147483647) where stickyClipGroupOrder IS NULL;"));
+				db.execDML(_T("Update Main set stickyClipOrder = -(2147483647) where stickyClipOrder = 0;"));
+				db.execDML(_T("Update Main set stickyClipGroupOrder = -(2147483647) where stickyClipGroupOrder = 0;"));
+
+				db.execDML(_T("CREATE INDEX Main_NoGroup ON Main(bIsGroup ASC, stickyClipOrder DESC, clipOrder DESC);"));
+				db.execDML(_T("CREATE INDEX Main_InGroup ON Main(lParentId ASC, bIsGroup ASC, stickyClipGroupOrder DESC, clipGroupOrder DESC);"));
+				db.execDML(_T("CREATE INDEX Data_ParentId_Format ON Data(lParentID COLLATE BINARY ASC, strClipBoardFormat COLLATE NOCASE ASC);"));
+			}
+		}
+		catch (CppSQLite3Exception& e)
+		{
+			if (didBackup == FALSE)
+				didBackup = BackupDB(csPath, backupFilePrefix);
+
+
 			e.errorCode();
 		}
 	}
@@ -492,6 +526,10 @@ BOOL CreateDB(CString csFile)
 				_T("DELETE FROM Data WHERE lParentID = old.clipID;\n")
 			_T("END\n"));
 
+		db.execDML(_T("CREATE INDEX Main_NoGroup ON Main(bIsGroup ASC, stickyClipOrder DESC, clipOrder DESC);"));
+		db.execDML(_T("CREATE INDEX Main_InGroup ON Main(lParentId ASC, bIsGroup ASC, stickyClipGroupOrder DESC, clipGroupOrder DESC);"));
+		db.execDML(_T("CREATE INDEX Data_ParentId_Format ON Data(lParentID COLLATE BINARY ASC, strClipBoardFormat COLLATE NOCASE ASC);"));
+
 		db.close();
 	}
 	CATCH_SQLITE_EXCEPTION_AND_RETURN(FALSE)

MainFrm.cpp

@@ -112,6 +112,8 @@ int CMainFrame::OnCreate(LPCREATESTRUCT lpCreateStruct)
     Log(_T("Setting polling timer to track focus"));
     SetTimer(ACTIVE_WINDOW_TIMER, g_Opt.FocusWndTimerTimeout(), 0);
 
+	SetTimer(READ_RANDOM_DB_FILE, g_Opt.ReadRandomFileInterval() * 1000, 0);
+
     SetWindowText(_T("Ditto"));
 
     HICON hIcon = AfxGetApp()->LoadIcon(IDR_MAINFRAME);
@@ -563,6 +565,12 @@ void CMainFrame::OnTimer(UINT_PTR nIDEvent)
 				}
 			}
 			break;
+
+		case READ_RANDOM_DB_FILE:
+			{
+				m_thread.FireReadDbFile();
+			}
+		break;
     }
 
     CFrameWnd::OnTimer(nIDEvent);

MainFrm.h

@@ -16,6 +16,7 @@
 #define KEY_STATE_MODIFIERS				8
 #define ACTIVE_WINDOW_TIMER				9
 #define TEXT_ONLY_PASTE					11
+#define READ_RANDOM_DB_FILE				12
 
 class CMainFrame: public CFrameWnd
 {

MainFrmThread.cpp

@@ -62,9 +62,39 @@ void CMainFrmThread::OnEvent(int eventId, void *param)
 		case SAVE_REMOTE_CLIPS:
 			OnSaveRemoteClips();
 			break;
+		case READ_DB_FILE:
+			OnReadDbFile();
+			break;
     }
 }
 
+//try and keep our db file in windows cache by randomly reading some data
+//not sure if this does what i think it does but looking into issues with slow access on large dbs
+void CMainFrmThread::OnReadDbFile()
+{
+	double idle = IdleSeconds();
+
+	if (idle < CGetSetOptions::ReadRandomFileIdleMin())
+	{
+		CString dbFile = CGetSetOptions::GetDBPath();
+		__int64 dbSize = FileSize(dbFile);
+
+		srand(time(NULL));
+
+		int random = rand() % (dbSize - 1024) + 1;
+
+		CFile f;
+		if (f.Open(dbFile, CFile::modeRead | CFile::shareDenyNone))
+		{
+			f.Seek(random, 0);
+			char data[1024];
+			f.Read(&data, 1024);
+
+			f.Close();
+		}
+	}
+}
+
 void CMainFrmThread::OnDeleteEntries()
 {
     RemoveOldEntries(true);

MainFrmThread.h

@@ -16,12 +16,14 @@ public:
         REMOVE_REMOTE_FILES, 
 		SAVE_CLIPS,
 		SAVE_REMOTE_CLIPS,
+		READ_DB_FILE,
 
         ECMAINFRMTHREADEVENTS_COUNT  //must be last
     };
 
     void FireDeleteEntries() { FireEvent(DELETE_ENTRIES); }
     void FireRemoveRemoteFiles() { FireEvent(REMOVE_REMOTE_FILES); }
+	void FireReadDbFile() { FireEvent(READ_DB_FILE); }
 
 	void AddClipToSave(CClip *pClip);
 	void AddRemoteClipToSave(CClipList *pClipList);
@@ -33,6 +35,7 @@ protected:
     void OnRemoveRemoteFiles();
 	void OnSaveClips();
 	void OnSaveRemoteClips();
+	void OnReadDbFile();
 
 	CCriticalSection m_cs;
 	CClipList m_saveClips;

Misc.h

@@ -12,6 +12,8 @@
 #define ONE_HOUR				3600000
 #define ONE_DAY					86400000
 
+#define INVALID_STICKY	-(2147483647)
+
 #define CATCH_SQLITE_EXCEPTION		\
 	catch (CppSQLite3Exception& e)	\
     {								\

Options.cpp

@@ -2197,4 +2197,24 @@ void CGetSetOptions::SetRequestFilesUsingIP(int val)
 int CGetSetOptions::GetRequestFilesUsingIP()
 {
 	return GetProfileLong(_T("RequestFilesUsingIP"), 1);
+}
+
+int CGetSetOptions::ReadRandomFileInterval()
+{
+	return GetProfileLong(_T("ReadRandomFileInterval"), 60);
+}
+
+int CGetSetOptions::ReadRandomFileIdleMin()
+{
+	return GetProfileLong(_T("ReadRandomFileIdleMin"), 30);
+}
+
+BOOL CGetSetOptions::GetShowGroupsInMainList()
+{
+	return GetProfileLong(_T("ShowGroupsInMainList"), 0);
+}
+
+void CGetSetOptions::SetShowGroupsInMainList(BOOL val)
+{
+	SetProfileLong(_T("ShowGroupsInMainList"), val);
 }

Options.h

@@ -474,6 +474,12 @@ public:
 
 	static void SetRequestFilesUsingIP(int val);
 	static int GetRequestFilesUsingIP();
+
+	static int ReadRandomFileInterval();
+	static int ReadRandomFileIdleMin();
+
+	static BOOL GetShowGroupsInMainList();
+	static void SetShowGroupsInMainList(BOOL val);
 };
 
 // global for easy access and for initialization of fast access variables

QPasteWnd.cpp

@@ -19,6 +19,7 @@
 #include "CreateQRCodeImage.h"
 #include "ClipCompare.h"
 //#include "MyDropTarget.h"
+#include "Misc.h"
 
 #ifdef _DEBUG
     #define new DEBUG_NEW
@@ -1018,13 +1019,22 @@ BOOL CQPasteWnd::FillList(CString csSQLSearch /*=""*/)
     {
         m_lstHeader.m_bStartTop = true;
         
-		csSort = "case when (Main.stickyClipOrder = 0 OR Main.stickyClipOrder IS NULL) then -9999999 else Main.stickyClipOrder END DESC, "
-				 "Main.bIsGroup ASC, "
-				 "clipOrder DESC";
+		//do not change this this directly relates to the views in the Main table
+		csSort = "Main.bIsGroup ASC, " 
+				 "Main.stickyClipOrder DESC, "				 
+				 "Main.clipOrder DESC";
 
         if(g_Opt.m_bShowAllClipsInMainList)
         {
-            strFilter = "((Main.bIsGroup = 1 AND Main.lParentID = -1) OR Main.bIsGroup = 0)";
+			if (CGetSetOptions::GetShowGroupsInMainList())
+			{
+				//found to be slower on large databases
+				strFilter = "((Main.bIsGroup = 1 AND Main.lParentID = -1) OR Main.bIsGroup = 0)";
+			}
+			else
+			{
+				strFilter = "(Main.bIsGroup = 0)";
+			}
         }
         else
         {
@@ -1036,9 +1046,10 @@ BOOL CQPasteWnd::FillList(CString csSQLSearch /*=""*/)
     {
         m_lstHeader.m_bStartTop = true;
 
-		csSort = "case when (Main.stickyClipGroupOrder = 0 OR Main.stickyClipGroupOrder IS NULL) then -9999999 else Main.stickyClipGroupOrder END DESC, "
-				 "Main.bIsGroup ASC, "
-				 "clipGroupOrder DESC";
+		//do not change this this directly relates to the views in the Main table
+		csSort = "Main.bIsGroup ASC, "
+				 "Main.stickyClipGroupOrder DESC, "				 
+				 "Main.clipGroupOrder DESC";
 			
 			//Main.stickyClipGroupOrder DESC, Main.clipGroupOrder DESC";//
         
@@ -3771,8 +3782,7 @@ void CQPasteWnd::GetDispInfo(NMHDR *pNMHDR, LRESULT *pResult)
                             cs += "G";
                         }
 
-						if (m_listItems[pItem->iItem].m_stickyClipOrder > 0 || 
-							m_listItems[pItem->iItem].m_stickyClipGroupOrder)
+						if (m_listItems[pItem->iItem].m_stickyClipOrder != INVALID_STICKY)
 						{
 							cs += "Sticky";
 						}

QPasteWnd.h

@@ -51,12 +51,7 @@ public:
 	static bool SortDesc(const CMainTable& d1, const CMainTable& d2)
 	{
 		double d1StickyOrder = d1.m_stickyClipOrder;
-		if (d1StickyOrder == 0)
-			d1StickyOrder = -9999999.0;
-
-		double d2StickyOrder = d2.m_stickyClipOrder;
-		if (d2StickyOrder == 0)
-			d2StickyOrder = -9999999.0;
+		double d2StickyOrder = d2.m_stickyClipOrder;		
 
 		if (d1StickyOrder != d2StickyOrder)
 			return d1StickyOrder > d2StickyOrder;
@@ -70,12 +65,8 @@ public:
 	static bool GroupSortDesc(const CMainTable& d1, const CMainTable& d2)
 	{
 		double d1StickyOrder = d1.m_stickyClipGroupOrder;
-		if (d1StickyOrder == 0)
-			d1StickyOrder = -9999999.0;
 
 		double d2StickyOrder = d2.m_stickyClipGroupOrder;
-		if (d2StickyOrder == 0)
-			d2StickyOrder = -9999999.0;
 
 		if (d1StickyOrder != d2StickyOrder)
 			return d1StickyOrder > d2StickyOrder;

QPasteWndThread.cpp

@@ -136,17 +136,17 @@ void CQPasteWndThread::OnLoadItems(void *param)
 					{
 						ATL::CCritSecLock csLock(pasteWnd->m_CritSection.m_sect);
 
-						if(pos < pasteWnd->m_listItems.size())
+						if (pos < pasteWnd->m_listItems.size())
 						{
 							pasteWnd->m_listItems[pos] = table;
 						}
-						else if(pos == pasteWnd->m_listItems.size())
+						else if (pos == pasteWnd->m_listItems.size())
 						{
 							pasteWnd->m_listItems.push_back(table);
 						}
-						else if(pos > pasteWnd->m_listItems.size())
+						else if (pos > pasteWnd->m_listItems.size())
 						{
-							for(int toAdd = pasteWnd->m_listItems.size(); toAdd < pos-1; toAdd++)
+							for (int toAdd = pasteWnd->m_listItems.size(); toAdd < pos - 1; toAdd++)
 							{
 								CMainTable empty;
 								empty.m_lID = -1;
@@ -154,7 +154,7 @@ void CQPasteWndThread::OnLoadItems(void *param)
 							}
 
 							pasteWnd->m_listItems.push_back(table);
-						}						
+						}
 					}
 
 					if(pasteWnd->m_bStopQuery)
@@ -175,13 +175,24 @@ void CQPasteWndThread::OnLoadItems(void *param)
 					pos++;
 				}
 
+				DWORD loadCount = GetTickCount() - startTick;
+				DWORD countCountStart = GetTickCount();
+				DWORD countCount = 0;
+				DWORD acceleratorCount = 0;
+
 				if(firstLoad)
 				{
 					::PostMessage(pasteWnd->m_hWnd, NM_REFRESH_ROW, -2, 0);
 					//allow the next thread message to process, this should be the message to set the list count
 
 					OnSetListCount(param);
+					
+					countCount = GetTickCount() - countCountStart;
+					DWORD acceleratorCountStart = GetTickCount();
+					 
 					OnLoadAccelerators(param);
+
+					acceleratorCount = GetTickCount() - acceleratorCountStart;
 				}
 				else
 				{
@@ -195,7 +206,7 @@ void CQPasteWndThread::OnLoadItems(void *param)
 					pasteWnd->m_loadItems.erase(pasteWnd->m_loadItems.begin());
 				}
 
-				Log(StrF(_T("Load items End count = %d, time = %d"), loadCount, GetTickCount() - startTick));
+				Log(StrF(_T("Load items End count = %d, Total Time = %d, LoadItems: %d, Count: %d, Accel: %d"), loadCount, GetTickCount() - startTick, loadCount, countCount, acceleratorCount));
 			}
 			catch (CppSQLite3Exception& e)	\
 			{								\

sqlite/CppSQLite3.cpp

@@ -977,9 +977,9 @@ sqlite3_stmt* CppSQLite3DB::compile(const TCHAR* szSQL)
 	sqlite3_stmt* pVM;
 
 #ifdef _UNICODE
-	int nRet = sqlite3_prepare16(mpDB, szSQL, -1, &pVM, (const void**)szTail);
+	int nRet = sqlite3_prepare16_v2(mpDB, szSQL, -1, &pVM, (const void**)szTail);
 #else
-	int nRet = sqlite3_prepare(mpDB, szSQL, -1, &pVM, &szTail);
+	int nRet = sqlite3_prepare_v2(mpDB, szSQL, -1, &pVM, &szTail);
 #endif
 
 	if (nRet != SQLITE_OK)