perf: importing graph

There're two things makes the importing slow:
1. db transact! for each file, now it's batched for 100 files.
2. block waiting for 10s before parsing so that the progress bar ui
can be changed, this is batched too for 10 files.

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -8,20 +8,13 @@
             [clojure.string :as string]
             [clojure.set :as set]))
 
-(defn- db-set-file-content!
-  "Modified copy of frontend.db.model/db-set-file-content!"
-  [conn path content]
-  (let [tx-data {:file/path path
-                 :file/content content}]
-    (d/transact! conn [tx-data] {:skip-refresh? true})))
-
 (defn parse-file
   "Parse file and save parsed data to the given db. Main parse fn used by logseq app"
-  [conn file content {:keys [new? delete-blocks-fn extract-options]
+  [conn file content {:keys [new? delete-blocks-fn extract-options skip-db-transact?]
                       :or {new? true
-                           delete-blocks-fn (constantly [])}
+                           delete-blocks-fn (constantly [])
+                           skip-db-transact? false}
                       :as options}]
-  (db-set-file-content! conn file content)
   (let [format (gp-util/get-format file)
         file-content [{:file/path file}]
         {:keys [tx ast]}
@@ -48,12 +41,15 @@
                {:tx (concat file-content pages-index delete-blocks pages block-ids blocks)
                 :ast ast})
              {:tx file-content})
-        tx (concat tx [(cond-> {:file/path file}
+        tx (concat tx [(cond-> {:file/path file
+                                :file/content content}
                                new?
                                ;; TODO: use file system timestamp?
                          (assoc :file/created-at (date-time-util/time-ms)))])
         tx' (gp-util/remove-nils tx)
-        result (d/transact! conn tx' (select-keys options [:new-graph? :from-disk?]))]
+        result (if skip-db-transact?
+                 tx'
+                 (d/transact! conn tx' (select-keys options [:new-graph? :from-disk?])))]
     {:tx result
      :ast ast}))
 

src/main/frontend/handler/common/file.cljs

@@ -62,22 +62,18 @@
                   :else
                   file)
            file (gp-util/path-normalize file)
-           new? (nil? (db/entity [:file/path file]))]
-       (:tx
-        (graph-parser/parse-file
-         (db/get-db repo-url false)
-         file
-         content
-         (merge (dissoc options :verbose)
-                {:new? new?
-                 :delete-blocks-fn (partial get-delete-blocks repo-url)
-                 :extract-options (merge
-                                   {:user-config (state/get-config)
-                                    :date-formatter (state/get-date-formatter)
-                                    :page-name-order (state/page-name-order)
-                                    :block-pattern (config/get-block-pattern (gp-util/get-format file))
-                                    :supported-formats (gp-config/supported-formats)}
-                                   (when (some? verbose) {:verbose verbose}))}))))
+           new? (nil? (db/entity [:file/path file]))
+           options (merge (dissoc options :verbose)
+                          {:new? new?
+                           :delete-blocks-fn (partial get-delete-blocks repo-url)
+                           :extract-options (merge
+                                             {:user-config (state/get-config)
+                                              :date-formatter (state/get-date-formatter)
+                                              :page-name-order (state/page-name-order)
+                                              :block-pattern (config/get-block-pattern (gp-util/get-format file))
+                                              :supported-formats (gp-config/supported-formats)}
+                                             (when (some? verbose) {:verbose verbose}))})]
+       (:tx (graph-parser/parse-file (db/get-db repo-url false) file content options)))
      (catch :default e
        (prn "Reset file failed " {:file file})
        (log/error :exception e)))))

src/main/frontend/handler/file.cljs

@@ -86,7 +86,8 @@
 
 ;; TODO: Remove this function in favor of `alter-files`
 (defn alter-file
-  [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose]
+  [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose
+                             skip-db-transact?]
                       :or {reset? true
                            re-render-root? false
                            from-disk? false
@@ -101,17 +102,19 @@
                                         (assoc (when original-content {:old-content original-content})
                                                :skip-compare? skip-compare?))))
         opts {:new-graph? new-graph?
-              :from-disk? from-disk?}]
-    (if reset?
-      (do
-        (when-let [page-id (db/get-file-page-id path)]
-          (db/transact! repo
-                        [[:db/retract page-id :block/alias]
-                         [:db/retract page-id :block/tags]]
-                        opts))
-        (file-common-handler/reset-file! repo path content (merge opts
-                                                                  (when (some? verbose) {:verbose verbose}))))
-      (db/set-file-content! repo path content opts))
+              :from-disk? from-disk?
+              :skip-db-transact? skip-db-transact?}
+        result (if reset?
+                 (do
+                   (when-not skip-db-transact?
+                     (when-let [page-id (db/get-file-page-id path)]
+                       (db/transact! repo
+                         [[:db/retract page-id :block/alias]
+                          [:db/retract page-id :block/tags]]
+                         opts)))
+                   (file-common-handler/reset-file! repo path content (merge opts
+                                                         (when (some? verbose) {:verbose verbose}))))
+                 (db/set-file-content! repo path content opts))]
     (util/p-handle (write-file!)
                    (fn [_]
                      (cond
@@ -134,7 +137,8 @@
                                           :status :error}]))
 
                      (println "Write file failed, path: " path ", content: " content)
-                     (log/error :write/failed error)))))
+                     (log/error :write/failed error)))
+    result))
 
 (defn set-file-content!
   [repo path new-content]

src/main/frontend/handler/repo.cljs

@@ -31,7 +31,8 @@
             [cljs-bean.core :as bean]
             [clojure.core.async :as async]
             [frontend.encrypt :as encrypt]
-            [frontend.mobile.util :as mobile-util]))
+            [frontend.mobile.util :as mobile-util]
+            [medley.core :as medley]))
 
 ;; Project settings should be checked in two situations:
 ;; 1. User changes the config.edn directly in logseq.com (fn: alter-file)
@@ -174,21 +175,27 @@
         file-paths [path]]
     (load-pages-metadata! repo file-paths files force?)))
 
+(defonce *file-tx (atom nil))
+
 (defn- parse-and-load-file!
-  [repo-url file {:keys [new-graph? verbose]}]
+  [repo-url file {:keys [new-graph? verbose skip-db-transact?]
+                  :or {skip-db-transact? true}}]
   (try
-    (file-handler/alter-file repo-url
-                             (:file/path file)
-                             (:file/content file)
-                             (merge {:new-graph? new-graph?
-                                     :re-render-root? false
-                                     :from-disk? true}
-                                    (when (some? verbose) {:verbose verbose})))
+    (reset! *file-tx
+            (file-handler/alter-file repo-url
+                                     (:file/path file)
+                                     (:file/content file)
+                                     (merge {:new-graph? new-graph?
+                                             :re-render-root? false
+                                             :from-disk? true
+                                             :skip-db-transact? skip-db-transact?}
+                                            (when (some? verbose) {:verbose verbose}))))
     (catch :default e
       (state/set-parsing-state! (fn [m]
                                   (update m :failed-parsing-files conj [(:file/path file) e])))))
   (state/set-parsing-state! (fn [m]
-                              (update m :finished inc))))
+                              (update m :finished inc)))
+  @*file-tx)
 
 (defn- after-parse
   [repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan]
@@ -210,8 +217,11 @@
   (let [supported-files (graph-parser/filter-files files)
         delete-data (->> (concat delete-files delete-blocks)
                          (remove nil?))
-        chan (async/to-chan! supported-files)
-        graph-added-chan (async/promise-chan)]
+        indexed-files (medley/indexed supported-files)
+        chan (async/to-chan! indexed-files)
+        graph-added-chan (async/promise-chan)
+        total (count supported-files)
+        large-graph? (> total 1000)]
     (when (seq delete-data) (db/transact! repo-url delete-data))
     (state/set-current-repo! repo-url)
     (state/set-parsing-state! {:total (count supported-files)})
@@ -220,18 +230,33 @@
       (do
         (doseq [file supported-files]
           (state/set-parsing-state! (fn [m]
-                                      (assoc m :current-parsing-file (:file/path file))))
-          (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose])))
+                                      (assoc m
+                                             :current-parsing-file (:file/path file))))
+          (parse-and-load-file! repo-url file (assoc
+                                               (select-keys opts [:new-graph? :verbose])
+                                               :skip-db-transact? false)))
         (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))
-      (async/go-loop []
-        (if-let [file (async/<! chan)]
-          (do
+      (async/go-loop [tx []]
+        (if-let [item (async/<! chan)]
+          (let [[idx file] item
+                yield-for-ui? (or (not large-graph?)
+                                  (zero? (rem idx 10))
+                                  (<= (- total idx) 10))]
             (state/set-parsing-state! (fn [m]
                                         (assoc m :current-parsing-file (:file/path file))))
-            (async/<! (async/timeout 10))
-            (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose]))
-            (recur))
-          (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan))))
+
+            (when yield-for-ui? (async/<! (async/timeout 1)))
+
+            (let [result (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose]))
+                  tx' (concat tx result)
+                  tx' (if (zero? (rem (inc idx) 100))
+                        (do (db/transact! repo-url tx' {:from-disk? true})
+                            [])
+                        tx')]
+              (recur tx')))
+          (do
+            (when (seq tx) (db/transact! repo-url tx {:from-disk? true}))
+            (after-parse repo-url files file-paths db-encrypted? re-render? re-render-opts opts graph-added-chan)))))
     graph-added-chan))
 
 (defn- parse-files-and-create-default-files!