partition load initial pages data

Also, add a script to build a test graph

src/dev-cljs/shadow/build_large_graph.cljs

@@ -0,0 +1,59 @@
+(ns shadow.build-large-graph)
+
+(comment
+
+  (in-ns 'frontend.db-worker)
+  (def repo "logseq_db_large-db-demo")
+  (def conn (worker-state/get-datascript-conn repo))
+
+  (defonce *ids (atom (set (map :v (d/datoms @conn :avet :block/uuid)))))
+  (defn get-next-id
+    []
+    (let [id (random-uuid)]
+      (if (@*ids id)
+        (get-next-id)
+        (do
+          (swap! *ids conj id)
+          id))))
+
+  (defn pages
+    [start-idx n]
+    (let [ids (repeatedly n get-next-id)]
+      (map-indexed
+       (fn [idx id]
+         {:block/uuid id
+          :block/original-name (str "page-" (+ start-idx idx))
+          :block/name (str "page-" (+ start-idx idx))
+          :block/format :markdown})
+       ids)))
+
+  (defn blocks
+    [page-id size]
+    (let [page-id [:block/uuid page-id]
+          blocks (vec (repeatedly size (fn []
+                                         (let [id (get-next-id)]
+                                           {:block/uuid id
+                                            :block/content (str id)
+                                            :block/format :markdown
+                                            :block/page page-id
+                                            :block/parent page-id}))))]
+      (map-indexed
+       (fn [i b]
+         (if (zero? i)
+           (assoc b :block/left page-id)
+           (let [left (nth blocks (dec i))]
+             (assoc b :block/left [:block/uuid (:block/uuid left)]))))
+       blocks)))
+
+  (defn create-graph!
+    [conn page-size blocks-size start-idx]
+    (let [pages (pages start-idx page-size)
+          page-blocks (map (fn [p]
+                             (cons p
+                                   (blocks (:block/uuid p) blocks-size))) pages)]
+      (doseq [data (partition-all 1000 page-blocks)]
+        (let [tx-data (apply concat data)]
+          (prn :debug :progressing (:block/name (first tx-data)))
+          (d/transact! conn tx-data {:new-graph? true})))))
+
+  (create-graph! conn 10000 20 0))

src/main/frontend/db_worker.cljs

@@ -30,7 +30,6 @@
 (defonce *datascript-conns worker-state/*datascript-conns)
 (defonce *opfs-pools worker-state/*opfs-pools)
 (defonce *publishing? (atom false))
-(defonce *store-jobs (atom #{}))
 
 (defn- get-pool-name
   [graph-name]
@@ -108,11 +107,8 @@
                   (fn [[addr data]]
                     #js {:$addr addr
                          :$content (pr-str data)})
-                  addr+data-seq)
-            p (p/do! (upsert-addr-content! repo data delete-addrs))]
-        (swap! *store-jobs conj p)
-        (p/then p (fn [] (swap! *store-jobs disj p)))
-        p))
+                  addr+data-seq)]
+        (upsert-addr-content! repo data delete-addrs)))
 
     (-restore [_ addr]
       (restore-data-from-addr repo addr))))
@@ -263,13 +259,7 @@
    [_this repo & {:keys [close-other-db?]
                   :or {close-other-db? true}}]
    (p/do!
-    ;; Store the current db if store jobs not finished yet
-    (when (seq @*store-jobs)
-      (-> (p/all @*store-jobs)
-          (p/then (fn [_]
-                    (reset! *store-jobs #{})
-                    (println "DB store job finished")))))
-    (when close-other-db?
+     (when close-other-db?
       (close-other-dbs! repo))
     (create-or-open-db! repo)))
 
@@ -374,12 +364,19 @@
      (let [data (->> (sqlite-common-db/get-initial-data @conn)
                      pr-str)]
        (async/go
+         ;; TODO: after UI db transacted
+         (async/<! (async/timeout 500))
          (let [all-pages (sqlite-common-db/get-all-pages @conn)
-               all-files (sqlite-common-db/get-all-files @conn)]
-           (worker-util/post-message :sync-db-changes (pr-str
-                                                       {:repo repo
-                                                        :tx-data (concat all-files all-pages)
-                                                        :tx-meta nil}))))
+               all-files (sqlite-common-db/get-all-files @conn)
+               full-data (concat all-files all-pages)
+               partitioned-data (map-indexed (fn [idx p] [idx p]) (partition-all 2000 full-data))]
+           (doseq [[idx tx-data] partitioned-data]
+             (worker-util/post-message :sync-db-changes (pr-str
+                                                         {:repo repo
+                                                          :tx-data tx-data
+                                                          :tx-meta {:initial-pages? true
+                                                                    :end? (= idx (dec (count partitioned-data)))}}))
+             (async/<! (async/timeout 100)))))
        data)))
 
   (closeDB

src/main/frontend/modules/outliner/pipeline.cljs

@@ -8,7 +8,8 @@
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.history :as history]
             [logseq.db :as ldb]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.util :as util]))
 
 (defn store-undo-data!
   [{:keys [tx-meta] :as opts}]
@@ -41,48 +42,53 @@
   ;;      :request-id request-id
   ;;      :tx-meta tx-meta
   ;;      :tx-data tx-data)
-  (let [{:keys [from-disk? new-graph? local-tx? undo? redo?]} tx-meta
+  (let [{:keys [from-disk? new-graph? local-tx? undo? redo? initial-pages? end?]} tx-meta
         repo (state/get-current-repo)
         tx-report {:tx-meta tx-meta
-                   :tx-data tx-data}]
-
-    (let [conn (db/get-db repo false)
-          tx-report (d/transact! conn tx-data tx-meta)]
-      (when local-tx?
-        (let [tx-id (get-tx-id tx-report)]
-          (store-undo-data! (assoc opts :tx-id tx-id))))
-      (when-not (or undo? redo?)
-        (update-current-tx-editor-cursor! tx-report)))
+                   :tx-data tx-data}
+        conn (db/get-db repo false)]
+    (if initial-pages?
+      (do
+        (util/profile "transact initial-pages" (d/transact! conn tx-data tx-meta))
+        (when end?
+          (ui-handler/re-render-root!)))
+      (do
+        (let [tx-report (d/transact! conn tx-data tx-meta)]
+          (when local-tx?
+            (let [tx-id (get-tx-id tx-report)]
+              (store-undo-data! (assoc opts :tx-id tx-id))))
+          (when-not (or undo? redo?)
+            (update-current-tx-editor-cursor! tx-report)))
 
-    (let [new-datoms (filter (fn [datom]
-                               (and
-                                (= :block/uuid (:a datom))
-                                (true? (:added datom)))) tx-data)]
-      (when (seq new-datoms)
-        (state/set-state! :editor/new-created-blocks (set (map :v new-datoms)))))
+        (let [new-datoms (filter (fn [datom]
+                                   (and
+                                    (= :block/uuid (:a datom))
+                                    (true? (:added datom)))) tx-data)]
+          (when (seq new-datoms)
+            (state/set-state! :editor/new-created-blocks (set (map :v new-datoms)))))
 
-    (if (or from-disk? new-graph?)
-      (do
-        (react/clear-query-state!)
-        (ui-handler/re-render-root!))
-      (when-not (:graph/importing @state/state)
-        (react/refresh! repo tx-report affected-keys)
+        (if (or from-disk? new-graph?)
+          (do
+            (react/clear-query-state!)
+            (ui-handler/re-render-root!))
+          (when-not (:graph/importing @state/state)
+            (react/refresh! repo tx-report affected-keys)
 
-        (when-let [state (:ui/restore-cursor-state @state/state)]
-          (when (or undo? redo?)
-            (restore-cursor-and-app-state! state undo?)
-            (state/set-state! :ui/restore-cursor-state nil)))
+            (when-let [state (:ui/restore-cursor-state @state/state)]
+              (when (or undo? redo?)
+                (restore-cursor-and-app-state! state undo?)
+                (state/set-state! :ui/restore-cursor-state nil)))
 
-        (state/set-state! :editor/start-pos nil)
+            (state/set-state! :editor/start-pos nil)
 
-        (when (and state/lsp-enabled?
-                   (seq blocks)
-                   (<= (count blocks) 1000))
-          (state/pub-event! [:plugin/hook-db-tx
-                             {:blocks  blocks
-                              :deleted-block-uuids deleted-block-uuids
-                              :tx-data (:tx-data tx-report)
-                              :tx-meta (:tx-meta tx-report)}]))))
+            (when (and state/lsp-enabled?
+                       (seq blocks)
+                       (<= (count blocks) 1000))
+              (state/pub-event! [:plugin/hook-db-tx
+                                 {:blocks  blocks
+                                  :deleted-block-uuids deleted-block-uuids
+                                  :tx-data (:tx-data tx-report)
+                                  :tx-meta (:tx-meta tx-report)}]))))))
 
     (when (= (:outliner-op tx-meta) :delete-page)
       (state/pub-event! [:page/deleted repo (:deleted-page tx-meta) (:file-path tx-meta) tx-meta]))