Merge pull request #10933 from logseq/perf/lazy-load-data

Perf enhancement: load partial data for UI needs

.carve/ignore

@@ -46,10 +46,6 @@ frontend.image/get-orientation
 frontend.mixins/perf-measure-mixin
 ;; Previously useful fn
 frontend.mobile.util/get-idevice-statusbar-height
-;; Used in macro
-frontend.modules.outliner.datascript/transact!
-frontend.modules.outliner.core/*transaction-opts*
-frontend.modules.outliner.core/*transaction-args*
 ;; Referenced in comment
 frontend.page/route-view
 ;; placeholder fn

.clj-kondo/config.edn

@@ -102,7 +102,7 @@
              frontend.idb idb
              frontend.loader loader
              frontend.mixins mixins
-             frontend.modules.outliner.core outliner-core
+             frontend.modules.outliner.ui ui-outliner-tx
              frontend.mobile.util mobile-util
              frontend.page page
              frontend.persist-db persist-db
@@ -127,9 +127,13 @@
              frontend.worker.handler.page.rename worker-page-rename
              frontend.worker.handler.file.util wfu
              lambdaisland.glogi log
-             logseq.common.path path
-             logseq.common.graph common-graph
              logseq.common.config common-config
+             logseq.common.graph common-graph
+             logseq.common.date-time-util date-time-util
+             logseq.common.path path
+             logseq.common.util common-util
+             logseq.common.util.page-ref page-ref
+             logseq.common.util.block-ref block-ref
              logseq.db ldb
              logseq.db.frontend.property db-property
              logseq.db.frontend.property.type db-property-type
@@ -142,12 +146,10 @@
              logseq.graph-parser.text text
              logseq.graph-parser.block gp-block
              logseq.graph-parser.mldoc gp-mldoc
-             logseq.common.util common-util
              logseq.graph-parser.property gp-property
-             logseq.common.util.page-ref page-ref
-             logseq.common.util.block-ref block-ref
              logseq.graph-parser.util.db db-util
-             logseq.graph-parser.date-time-util date-time-util
+             logseq.outliner.core outliner-core
+             logseq.outliner.op outliner-op
              logseq.outliner.pipeline outliner-pipeline
              logseq.outliner.datascript-report ds-report
              medley.core medley
@@ -170,7 +172,6 @@
            clojure.test.check.clojure-test/defspec clojure.core/def
            clojure.test.check.properties/for-all clojure.core/for
            ;; src/main
-           frontend.modules.outliner.datascript/auto-transact! clojure.core/let
            frontend.namespaces/import-vars potemkin/import-vars
            ;; src/test
            frontend.test.helper/deftest-async clojure.test/deftest

deps/common/.carve/config.edn

@@ -3,6 +3,7 @@
                   logseq.common.util.page-ref
                   logseq.common.util.block-ref
                   logseq.common.util
+                  logseq.common.util.date-time
                   logseq.common.marker
                   logseq.common.config]
  :report {:format :ignore}}

deps/graph-parser/src/logseq/graph_parser/date_time_util.cljs → deps/common/src/logseq/common/util/date_time.cljs

@@ -1,5 +1,5 @@
-(ns logseq.graph-parser.date-time-util
-  "cljs-time util fns for graph-parser"
+(ns logseq.common.util.date-time
+  "cljs-time util fns for deps"
   (:require [cljs-time.format :as tf]
             [clojure.string :as string]
             [logseq.common.util :as common-util]))
@@ -73,3 +73,9 @@
   ([date sep]
    (let [{:keys [year month day]} (year-month-day-padded (get-date date))]
      (str year sep month sep day))))
+
+(defn date->int
+  "Given a date object, returns its journal page integer"
+  [date]
+  (parse-long
+   (string/replace (ymd date) "/" "")))

deps/db/src/logseq/db.cljs

@@ -70,6 +70,14 @@
                (when-let [callback (:callback (get new request-id))]
                  (callback)))))
 
+(defn get-next-request-id
+  []
+  (swap! *request-id inc))
+
+(defn add-request!
+  [request-id data]
+  (swap! *request-id->response assoc request-id (if (map? data) data {:response data})))
+
 (defn transact!
   "`repo-or-conn`: repo for UI thread and conn for worker/node"
   ([repo-or-conn tx-data]
@@ -87,7 +95,7 @@
 
        (let [f (or @*transact-fn d/transact!)
              sync? (= f d/transact!)
-             request-id (when-not sync? (swap! *request-id inc))
+             request-id (when-not sync? (get-next-request-id))
              tx-meta' (cond-> tx-meta
                         (not sync?)
                         (assoc :request-id request-id))]
@@ -100,7 +108,7 @@
                            {:response resp}
                            {:response resp
                             :callback #(f repo-or-conn tx-data tx-meta')})]
-               (swap! *request-id->response assoc request-id value))
+               (add-request! request-id value))
              resp)))))))
 
 (defn build-default-pages-tx
@@ -342,6 +350,7 @@
         parents))))
 
 (defn get-block-children-ids
+  "Returns children UUIDs"
   [db block-uuid]
   (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
     (let [seen   (volatile! [])]
@@ -527,6 +536,60 @@
                      '[:db/id :block/name :block/original-name]
                      ids)))))
 
+(defn get-page-alias
+  [db page-id]
+  (->>
+   (d/q
+    '[:find [?e ...]
+      :in $ ?page %
+      :where
+      (alias ?page ?e)]
+    db
+    page-id
+    (:alias rules/rules))
+   distinct))
+
+(defn get-page-refs
+  [db id]
+  (let [alias (->> (get-page-alias db id)
+                   (cons id)
+                   distinct)
+        refs (->> (mapcat (fn [id] (:block/_path-refs (d/entity db id))) alias)
+                  distinct)]
+    (when (seq refs)
+      (d/pull-many db '[*] (map :db/id refs)))))
+
+(defn get-block-refs
+  [db id]
+  (let [block (d/entity db id)]
+    (if (:block/name block)
+      (get-page-refs db id)
+      (let [refs (:block/_refs (d/entity db id))]
+        (when (seq refs)
+          (d/pull-many db '[*] (map :db/id refs)))))))
+
+(defn get-block-refs-count
+  [db id]
+  (some-> (d/entity db id)
+          :block/_refs
+          count))
+
+(defn get-page-unlinked-refs
+  "Get unlinked refs from search result"
+  [db page-id search-result-eids]
+  (let [alias (->> (get-page-alias db page-id)
+                   (cons page-id)
+                   set)
+        eids (remove
+              (fn [eid]
+                (when-let [e (d/entity db eid)]
+                  (or (some alias (map :db/id (:block/refs e)))
+                      (:block/link e)
+                      (nil? (:block/content e)))))
+              search-result-eids)]
+    (when (seq eids)
+      (d/pull-many db '[*] eids))))
+
 (comment
   (defn db-based-graph?
     "Whether the current graph is db-only"

deps/db/src/logseq/db/frontend/schema.cljs

@@ -113,6 +113,8 @@
    :file/path {:db/unique :db.unique/identity}
    ;; only store the content of logseq's files
    :file/content {}
+
+   ;; TODO: do we really use this?
    :file/handle {}
    ;; :file/created-at {}
    ;; :file/last-modified-at {}

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -3,18 +3,140 @@
   (:require [datascript.core :as d]
             ["path" :as node-path]
             [clojure.string :as string]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.common.util.date-time :as date-time-util]
+            [logseq.common.util :as common-util]))
+
+(comment
+  (defn- get-built-in-files
+    [db]
+    (let [files ["logseq/config.edn"
+                 "logseq/custom.css"
+                 "logseq/custom.js"]]
+      (map #(d/pull db '[*] [:file/path %]) files))))
+
+(defn get-all-pages
+  [db]
+  (->> (d/datoms db :avet :block/name)
+       (map (fn [e]
+              (d/pull db '[*] (:e e))))))
+
+(defn get-all-files
+  [db]
+  (->> (d/datoms db :avet :file/path)
+       (map (fn [e]
+              {:db/id (:e e)
+               :file/path (:v e)
+               :file/content (:file/content (d/entity db (:e e)))}))))
+
+(defn- with-block-refs
+  [db block]
+  (update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
+
+(defn with-parent-and-left
+  [db block]
+  (cond
+    (:block/name block)
+    block
+    (:block/page block)
+    (let [left (when-let [e (d/entity db (:db/id (:block/left block)))]
+                 (select-keys e [:db/id :block/uuid]))
+          parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
+                   (select-keys e [:db/id :block/uuid]))]
+      (->>
+       (assoc block
+              :block/left left
+              :block/parent parent)
+       (common-util/remove-nils-non-nested)
+       (with-block-refs db)))
+    :else
+    block))
+
+(defn- with-tags
+  [db block]
+  (update block :block/tags (fn [tags] (d/pull-many db '[*] (map :db/id tags)))))
+
+(defn- mark-block-fully-loaded
+  [b]
+  (assoc b :block.temp/fully-loaded? true))
+
+(defn get-block-and-children
+  [db name children?]
+  (let [uuid? (common-util/uuid-string? name)
+        block (when uuid?
+                (let [id (uuid name)]
+                  (d/entity db [:block/uuid id])))
+        get-children (fn [children]
+                       (let [long-page? (> (count children) 500)]
+                         (if long-page?
+                           (map (fn [e]
+                                  (select-keys e [:db/id :block/uuid :block/page :block/left :block/parent :block/collapsed?]))
+                                children)
+                           (->> (d/pull-many db '[*] (map :db/id children))
+                                (map #(with-block-refs db %))
+                                (map mark-block-fully-loaded)))))]
+    (if (and block (not (:block/name block))) ; not a page
+      (let [block' (->> (d/pull db '[*] (:db/id block))
+                        (with-parent-and-left db)
+                        (with-block-refs db)
+                        mark-block-fully-loaded)]
+        (cond->
+         {:block block'}
+          children?
+          (assoc :children (get-children (:block/_parent block)))))
+      (when-let [block (or block (d/entity db [:block/name name]))]
+        (cond->
+         {:block (->> (d/pull db '[*] (:db/id block))
+                      (with-tags db)
+                      mark-block-fully-loaded)}
+          children?
+          (assoc :children
+                 (if (contains? (:block/type block) "whiteboard")
+                   (->> (d/pull-many db '[*] (map :db/id (:block/_page block)))
+                        (map #(with-block-refs db %))
+                        (map mark-block-fully-loaded))
+                   (get-children (:block/_page block)))))))))
+
+(defn get-latest-journals
+  [db n]
+  (let [today (date-time-util/date->int (js/Date.))]
+    (->>
+     (d/q '[:find [(pull ?page [*]) ...]
+            :in $ ?today
+            :where
+            [?page :block/name ?page-name]
+            [?page :block/journal? true]
+            [?page :block/journal-day ?journal-day]
+            [(<= ?journal-day ?today)]]
+          db
+          today)
+     (sort-by :block/journal-day)
+     (reverse)
+     (take n))))
+
+(defn get-structured-blocks
+  [db]
+  (let [special-pages (map #(d/pull db '[*] [:block/name %]) #{"tags"})
+        closed-values (->> (d/datoms db :avet :block/type)
+                           (keep (fn [e]
+                                   (when (contains? #{"closed value"} (:v e))
+                                     (d/pull db '[*] (:e e))))))]
+    (concat special-pages closed-values)))
 
 (defn get-initial-data
-  "Returns initial data as vec of datoms"
+  "Returns initial data"
   [db]
-  (->> (d/datoms db :eavt)
-       vec))
+  (let [latest-journals (get-latest-journals db 3)
+        all-files (get-all-files db)
+        structured-blocks (get-structured-blocks db)]
+    (concat latest-journals all-files structured-blocks)))
 
 (defn restore-initial-data
   "Given initial sqlite data and schema, returns a datascript connection"
-  [datoms schema]
-  (d/conn-from-datoms datoms schema))
+  [data schema]
+  (let [conn (d/create-conn schema)]
+    (d/transact! conn data)
+    conn))
 
 (defn create-kvs-table!
   "Creates a sqlite table for use with datascript.storage if one doesn't exist"

deps/db/test/logseq/db/sqlite/db_test.cljs → deps/db/test/logseq/db/sqlite/common_db_test.cljs

@@ -1,11 +1,13 @@
-(ns logseq.db.sqlite.db-test
+(ns logseq.db.sqlite.common-db-test
   (:require [cljs.test :refer [deftest async use-fixtures is testing]]
             ["fs" :as fs]
             ["path" :as node-path]
             [datascript.core :as d]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
             [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.sqlite.db :as sqlite-db]))
+            [logseq.common.util.date-time :as date-time-util]
+            [logseq.db.sqlite.db :as sqlite-db]
+            [clojure.string :as string]))
 
 (use-fixtures
   :each
@@ -27,8 +29,7 @@
     (create-graph-dir "tmp/graphs" "test-db")
 
     (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
-          blocks [{:block/uuid (random-uuid)
-                   :file/path "logseq/config.edn"
+          blocks [{:file/path "logseq/config.edn"
                    :file/content "{:foo :bar}"}]
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
@@ -41,16 +42,20 @@
           "Correct file with content is found"))))
 
 (deftest restore-initial-data
-  (testing "Restore a journal page with its block"
+  (testing "Restore a journal page"
     (create-graph-dir "tmp/graphs" "test-db")
     (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
           page-uuid (random-uuid)
           block-uuid (random-uuid)
           created-at (js/Date.now)
+          date-int (date-time-util/date->int (js/Date.))
+          date-title (date-time-util/int->journal-title date-int "MMM do, yyyy")
           blocks [{:db/id 100001
                    :block/uuid page-uuid
-                   :block/journal-day 20230629
-                   :block/name "jun 29th, 2023"
+                   :block/journal? true
+                   :block/journal-day date-int
+                   :block/name (string/lower-case date-title)
+                   :block/original-name date-title
                    :block/created-at created-at
                    :block/updated-at created-at}
                   {:db/id 100002
@@ -63,9 +68,9 @@
           ;; Simulate getting data from sqlite and restoring it for frontend
           conn (-> (sqlite-common-db/get-initial-data @conn*)
                    (sqlite-common-db/restore-initial-data db-schema/schema-for-db-based-graph))]
-      (is (= blocks
+      (is (= (take 1 blocks)
              (->> (d/q '[:find (pull ?b [*])
                          :where [?b :block/created-at]]
                        @conn)
                   (map first)))
-          "Datascript db matches data inserted into sqlite"))))
+          "Journal page is included in initial restore while its block is not"))))

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -16,7 +16,7 @@
   (mapcat (fn [{uuid :block/uuid eid :db/id}]
             (if (and uuid (contains? retain-uuids uuid))
               (map (fn [attr] [:db.fn/retractAttribute eid attr]) db-schema/retract-attributes)
-              [[:db.fn/retractEntity eid]]))
+              (when eid [[:db.fn/retractEntity eid]])))
           blocks))
 
 (defn- get-file-page

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -5,7 +5,7 @@
             [clojure.walk :as walk]
             [datascript.core :as d]
             [logseq.common.config :as common-config]
-            [logseq.graph-parser.date-time-util :as date-time-util]
+            [logseq.common.util.date-time :as date-time-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.text :as text]

deps/graph-parser/src/logseq/graph_parser/util/db.cljs

@@ -2,7 +2,7 @@
   "Db util fns that are useful for the frontend and nbb-logseq. This may be used
   by the graph-parser soon but if not, it should be in its own library"
   (:require [cljs-time.core :as t]
-            [logseq.graph-parser.date-time-util :as date-time-util]
+            [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util :as common-util]
             [logseq.common.util.page-ref :as page-ref]
             [datascript.core :as d]
@@ -17,11 +17,7 @@ it will return 1622433600000, which is equivalent to Mon May 31 2021 00 :00:00."
   ([date hours mins secs millisecs]
    (.setHours (js/Date. date) hours mins secs millisecs)))
 
-(defn date->int
-  "Given a date object, returns its journal page integer"
-  [date]
-  (parse-long
-   (string/replace (date-time-util/ymd date) "/" "")))
+(def date->int date-time-util/date->int)
 
 (defn old->new-relative-date-format [input]
   (let [count (re-find #"^\d+" (name input))

deps/outliner/.carve/ignore

@@ -2,7 +2,7 @@
 logseq.outliner.cli.pipeline/add-listener
 ;; private
 logseq.outliner.core/*transaction-opts*
-;; private
-logseq.outliner.core/*transaction-args*
 ;; API fn
 logseq.outliner.datascript/transact!
+;; API fn
+logseq.outliner.op/apply-ops!

deps/outliner/src/logseq/outliner/core.cljs

@@ -152,7 +152,7 @@
                                   ;; Only delete if last reference
                                   (keep #(when (<= (count (:block/_macros (d/entity db (:db/id %))))
                                                    1)
-                                           (vector :db.fn/retractEntity (:db/id %)))
+                                           (when (:db/id %) (vector :db.fn/retractEntity (:db/id %))))
                                         (:block/macros block-entity)))))))
 
 (comment
@@ -233,7 +233,10 @@
     (let [refs (->> (rebuild-block-refs repo conn date-formatter block (:block/properties block)
                                         :skip-content-parsing? true)
                     (concat (:block/refs m))
-                    (concat (:block/tags m)))]
+                    (concat (if (seq (:block/tags m))
+                              (:block/tags m)
+                              (map :db/id (:block/tags (d/entity @conn [:block/uuid (:block/uuid block)])))))
+                    (remove nil?))]
       (swap! txs-state (fn [txs] (concat txs [{:db/id (:db/id block)
                                                :block/refs refs}]))))))
 
@@ -340,7 +343,7 @@
                   data)
           m (-> data'
                 (dissoc :block/children :block/meta :block.temp/top? :block.temp/bottom? :block/unordered
-                        :block/title :block/body :block/level)
+                        :block/title :block/body :block/level :block.temp/fully-loaded?)
                 common-util/remove-nils
                 block-with-updated-at
                 fix-tag-ids)
@@ -355,11 +358,12 @@
               (db-marker-handle conn))]
 
       ;; Ensure block UUID never changes
-      (when (and db-id block-uuid)
-        (let [uuid-not-changed? (= block-uuid (:block/uuid (d/entity db db-id)))]
-          (when-not uuid-not-changed?
-            (js/console.error "Block UUID shouldn't be changed once created"))
-          (assert uuid-not-changed? "Block UUID changed")))
+      (let [e (d/entity db db-id)]
+        (when (and e block-uuid)
+          (let [uuid-not-changed? (= block-uuid (:block/uuid e))]
+            (when-not uuid-not-changed?
+              (js/console.error "Block UUID shouldn't be changed once created"))
+            (assert uuid-not-changed? "Block UUID changed"))))
 
       (when eid
         ;; Retract attributes to prepare for tx which rewrites block attributes
@@ -398,11 +402,13 @@
     (assert (ds/outliner-txs-state? txs-state)
             "db should be satisfied outliner-tx-state?")
     (let [block-id (otree/-get-id this conn)
-          ids (set (if children?
-                     (let [children (ldb/get-block-children @conn block-id)
-                           children-ids (map :block/uuid children)]
-                       (conj children-ids block-id))
-                     [block-id]))
+          ids (->>
+               (if children?
+                 (let [children (ldb/get-block-children @conn block-id)
+                       children-ids (map :block/uuid children)]
+                   (conj children-ids block-id))
+                 [block-id])
+               (remove nil?))
           txs (map (fn [id] [:db.fn/retractEntity [:block/uuid id]]) ids)
           txs (if-not children?
                 (let [immediate-children (ldb/get-block-immediate-children @conn block-id)]
@@ -1050,7 +1056,7 @@
 
 (defn- ^:large-vars/cleanup-todo indent-outdent-blocks
   "Indent or outdent `blocks`."
-  [repo conn blocks indent? & {:keys [get-first-block-original logical-outdenting?]}]
+  [repo conn blocks indent? & {:keys [parent-original logical-outdenting?]}]
   {:pre [(seq blocks) (boolean? indent?)]}
   (let [db @conn
         top-level-blocks (map (fn [b] (d/entity db (:db/id b))) blocks)
@@ -1083,34 +1089,33 @@
                     (concat-tx-fn result collapsed-tx))
                   (move-blocks repo conn blocks' left (merge opts {:sibling? false
                                                                    :indent? true}))))))
-          (let [parent-original (when get-first-block-original (get-first-block-original))]
-            (if parent-original
+          (if parent-original
+            (let [blocks' (take-while (fn [b]
+                                        (not= (:db/id (:block/parent b))
+                                              (:db/id (:block/parent parent))))
+                                      top-level-blocks)]
+              (move-blocks repo conn blocks' parent-original (merge opts {:outliner-op :indent-outdent-blocks
+                                                                          :sibling? true
+                                                                          :indent? false})))
+
+            (when (and parent (not (page-block? (d/entity db (:db/id parent)))))
               (let [blocks' (take-while (fn [b]
                                           (not= (:db/id (:block/parent b))
                                                 (:db/id (:block/parent parent))))
-                                        top-level-blocks)]
-                (move-blocks repo conn blocks' parent-original (merge opts {:outliner-op :indent-outdent-blocks
-                                                                            :sibling? true
-                                                                            :indent? false})))
-
-              (when (and parent (not (page-block? (d/entity db (:db/id parent)))))
-                (let [blocks' (take-while (fn [b]
-                                            (not= (:db/id (:block/parent b))
-                                                  (:db/id (:block/parent parent))))
-                                          top-level-blocks)
-                      result (move-blocks repo conn blocks' parent (merge opts {:sibling? true}))]
-                  (if logical-outdenting?
-                    result
+                                        top-level-blocks)
+                    result (move-blocks repo conn blocks' parent (merge opts {:sibling? true}))]
+                (if logical-outdenting?
+                  result
                   ;; direct outdenting (default behavior)
-                    (let [last-top-block (d/entity db (:db/id (last blocks')))
-                          right-siblings (->> (get-right-siblings conn (block db last-top-block))
-                                              (map :data))]
-                      (if (seq right-siblings)
-                        (let [result2 (if-let [last-direct-child-id (ldb/get-block-last-direct-child-id db (:db/id last-top-block))]
-                                        (move-blocks repo conn right-siblings (d/entity db last-direct-child-id) (merge opts {:sibling? true}))
-                                        (move-blocks repo conn right-siblings last-top-block (merge opts {:sibling? false})))]
-                          (concat-tx-fn result result2))
-                        result))))))))))))
+                  (let [last-top-block (d/entity db (:db/id (last blocks')))
+                        right-siblings (->> (get-right-siblings conn (block db last-top-block))
+                                            (map :data))]
+                    (if (seq right-siblings)
+                      (let [result2 (if-let [last-direct-child-id (ldb/get-block-last-direct-child-id db (:db/id last-top-block))]
+                                      (move-blocks repo conn right-siblings (d/entity db last-direct-child-id) (merge opts {:sibling? true}))
+                                      (move-blocks repo conn right-siblings last-top-block (merge opts {:sibling? false})))]
+                        (concat-tx-fn result result2))
+                      result)))))))))))
 
 ;;; ### write-operations have side-effects (do transactions) ;;;;;;;;;;;;;;;;
 
@@ -1124,10 +1129,6 @@
   see also `logseq.outliner.transaction/transact!`"
   nil)
 
-(def ^:private ^:dynamic #_:clj-kondo/ignore *transaction-args*
-  "Stores transaction args which can be fetched in all op-transact functions."
-  nil)
-
 (defn- op-transact!
   [fn-var & args]
   {:pre [(var? fn-var)]}

deps/outliner/src/logseq/outliner/datascript.cljs

@@ -2,7 +2,6 @@
   "Provides fns related to wrapping datascript's transact!"
   (:require [logseq.common.util :as common-util]
             [logseq.common.util.block-ref :as block-ref]
-            [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.property :as gp-property]
             [datascript.core :as d]
             [clojure.string :as string]
@@ -65,7 +64,7 @@
                               ;; Only delete if last reference
                               (keep #(when (<= (count (:block/_macros (d/entity db (:db/id %))))
                                                1)
-                                       (vector :db.fn/retractEntity (:db/id %)))
+                                       (when (:db/id %) (vector :db.fn/retractEntity (:db/id %))))
                                     (:block/macros b)))
                             retracted-blocks)]
       (when (and (seq retracted-tx') (fn? set-state-fn))
@@ -76,9 +75,8 @@
     txs))
 
 (defn transact!
-  [txs tx-meta {:keys [repo conn unlinked-graph? set-state-fn]}]
-  (let [db-based? (and repo (sqlite-util/db-based-graph? repo))
-        txs (map (fn [m]
+  [txs tx-meta {:keys [repo conn set-state-fn]}]
+  (let [txs (map (fn [m]
                    (if (map? m)
                      (dissoc m :block/children :block/meta :block/top? :block/bottom? :block/anchor
                              :block/title :block/body :block/level :block/container :db/other-tx
@@ -92,10 +90,7 @@
               true
               (distinct))]
 
-    (when (and (seq txs)
-               (or db-based?
-                   (and (fn? unlinked-graph?) (not (unlinked-graph?)))
-                   (some? js/process)))
+    (when (seq txs)
 
       ;; (prn :debug "DB transact")
       ;; (cljs.pprint/pprint txs)

deps/outliner/src/logseq/outliner/op.cljs

@@ -0,0 +1,87 @@
+(ns logseq.outliner.op
+  "Transact outliner ops"
+  (:require [logseq.outliner.transaction :as outliner-tx]
+            [logseq.outliner.core :as outliner-core]
+            [datascript.core :as d]
+            [malli.core :as m]))
+
+(def op-schema
+  [:multi {:dispatch first}
+   [:save-block
+    [:catn
+     [:op :keyword]
+     [:args [:tuple ::block]]]]
+   [:insert-blocks
+    [:catn
+     [:op :keyword]
+     [:args [:tuple ::blocks ::id ::option]]]]
+   [:delete-blocks
+    [:catn
+     [:op :keyword]
+     [:args [:tuple ::ids ::option]]]]
+   [:move-blocks
+    [:catn
+     [:op :keyword]
+     [:args [:tuple ::ids ::id :boolean]]]]
+   [:move-blocks-up-down
+    [:catn
+     [:op :keyword]
+     [:args [:tuple ::ids :boolean]]]]
+   [:indent-outdent-blocks
+    [:catn
+     [:op :keyword]
+     [:args [:tuple ::ids :boolean ::option]]]]])
+
+(def ops-schema [:schema {:registry {::id int?
+                                     ::block map?
+                                     ::option [:maybe map?]
+                                     ::blocks [:sequential ::block]
+                                     ::ids [:sequential ::id]}}
+                 [:sequential op-schema]])
+
+(def ops-validator (m/validator ops-schema))
+
+(defn apply-ops!
+  [repo conn ops date-formatter opts]
+  (assert (ops-validator ops) ops)
+  (let [opts' (assoc opts
+                     :transact-opts {:repo repo :conn conn}
+                     :local-tx? true)
+        *insert-result (atom nil)]
+    (outliner-tx/transact!
+     opts'
+     (doseq [[op args] ops]
+       (case op
+         :save-block
+         (apply outliner-core/save-block! repo conn date-formatter args)
+
+         :insert-blocks
+         (let [[blocks target-block-id opts] args]
+           (when-let [target-block (d/entity @conn target-block-id)]
+             (let [result (outliner-core/insert-blocks! repo conn blocks target-block opts)]
+               (reset! *insert-result result))))
+
+         :delete-blocks
+         (let [[block-ids opts] args
+               blocks (keep #(d/entity @conn %) block-ids)]
+           (outliner-core/delete-blocks! repo conn date-formatter blocks opts))
+
+         :move-blocks
+         (let [[block-ids target-block-id sibling?] args
+               blocks (keep #(d/entity @conn %) block-ids)
+               target-block (d/entity @conn target-block-id)]
+           (when (and target-block (seq blocks))
+             (outliner-core/move-blocks! repo conn blocks target-block sibling?)))
+
+         :move-blocks-up-down
+         (let [[block-ids up?] args
+               blocks (keep #(d/entity @conn %) block-ids)]
+           (when (seq blocks)
+             (outliner-core/move-blocks-up-down! repo conn blocks up?)))
+
+         :indent-outdent-blocks
+         (let [[block-ids indent? opts] args
+               blocks (keep #(d/entity @conn %) block-ids)]
+           (when (seq blocks)
+             (outliner-core/indent-outdent-blocks! repo conn blocks indent? opts))))))
+    @*insert-result))

deps/outliner/src/logseq/outliner/transaction.cljc

@@ -44,8 +44,7 @@
                                  (get opts*# :persist-op? true)
                                  (assoc :persist-op? true))]
          (binding [logseq.outliner.core/*transaction-data* (transient [])
-                   logseq.outliner.core/*transaction-opts* (transient [])
-                   logseq.outliner.core/*transaction-args* transaction-args#]
+                   logseq.outliner.core/*transaction-opts* (transient [])]
            (conj! logseq.outliner.core/*transaction-opts* opts#)
            ~@body
            (let [r# (persistent! logseq.outliner.core/*transaction-data*)

deps/shui/src/logseq/shui/list_item/v1.cljs

@@ -58,7 +58,7 @@
              (recur more))))))
 
 (defn highlight-query* [app-config query text]
-  (if (vector? text)                    ; hiccup
+  (if (or (vector? text) (object? text))                    ; hiccup
     text
     (let [text-string (to-string text)]
       (if-not (seq text-string)

scripts/README.md

@@ -31,6 +31,25 @@ properties. Read the docs in
 [logseq.tasks.db-graph.create-graph](src/logseq/tasks/db_graph/create_graph.cljs)
 for specifics on the EDN map.
 
+To create large graphs with varying size:
+
+```
+$ yarn -s nbb-logseq src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs large
+Building tx ...
+Built 21000 tx, 1000 pages and 20000 blocks ...
+Transacting chunk 1 of 21 starting with block: #:block{:name "page-0"}
+...
+Created graph large with 187810 datoms!
+
+# To see options available
+$ yarn -s nbb-logseq src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs -h
+Usage: $0 GRAPH-NAME [OPTIONS]
+Options:
+  -h, --help        Print help
+  -p, --pages  1000 Number of pages to create
+  -b, --blocks 20   Number of blocks to create
+```
+
 Another example is the `create_graph_with_schema_org.cljs` script which creates a graph
 with the https://schema.org/ ontology with as many of the classes and properties as possible:
 

scripts/src/logseq/tasks/db_graph/create_graph_with_large_sizes.cljs

@@ -0,0 +1,84 @@
+(ns logseq.tasks.db-graph.create-graph-with-large-sizes
+  "Script that generates graphs at large sizes"
+  (:require [logseq.tasks.db-graph.create-graph :as create-graph]
+            [clojure.string :as string]
+            [datascript.core :as d]
+            [babashka.cli :as cli]
+            ["path" :as node-path]
+            ["os" :as os]
+            [nbb.core :as nbb]))
+
+(def *ids (atom #{}))
+(defn get-next-id
+  []
+  (let [id (random-uuid)]
+    (if (@*ids id)
+      (get-next-id)
+      (do
+        (swap! *ids conj id)
+        id))))
+
+(defn build-pages
+  [start-idx n]
+  (let [ids (repeatedly n get-next-id)]
+    (map-indexed
+     (fn [idx id]
+       {:block/uuid id
+        :block/name (str "page-" (+ start-idx idx))})
+     ids)))
+
+(defn build-blocks
+  [size]
+  (vec (repeatedly size
+                   (fn []
+                     (let [id (get-next-id)]
+                       {:block/uuid id
+                        :block/content (str id)})))))
+
+(defn- create-init-data
+  [options]
+  (let [pages (build-pages 0 (:pages options))]
+    {:pages-and-blocks
+     (mapv #(hash-map :page % :blocks (build-blocks (:blocks options)))
+           pages)}))
+
+(def spec
+  "Options spec"
+  {:help {:alias :h
+          :desc "Print help"}
+   :pages {:alias :p
+           :default 1000
+           :desc "Number of pages to create"}
+   :blocks {:alias :b
+            :default 20
+            :desc "Number of blocks to create"}})
+
+(defn -main [args]
+  (let [graph-dir (first args)
+        options (cli/parse-opts args {:spec spec})
+        _ (when (or (nil? graph-dir) (:help options))
+            (println (str "Usage: $0 GRAPH-NAME [OPTIONS]\nOptions:\n"
+                          (cli/format-opts {:spec spec})))
+            (js/process.exit 1))
+        [dir db-name] (if (string/includes? graph-dir "/")
+                        ((juxt node-path/dirname node-path/basename) graph-dir)
+                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
+        conn (create-graph/init-conn dir db-name)
+        _ (println "Building tx ...")
+        blocks-tx (create-graph/create-blocks-tx (create-init-data options))]
+    (println "Built" (count blocks-tx) "tx," (count (filter :block/name blocks-tx)) "pages and"
+             (count (filter :block/content blocks-tx)) "blocks ...")
+    ;; Vary the chunking with page size up to a max to avoid OOM
+    (let [tx-chunks (partition-all (min (:pages options) 30000) blocks-tx)]
+      (loop [chunks tx-chunks
+             chunk-num 1]
+        (when-let [chunk (first chunks)]
+          (println "Transacting chunk" chunk-num  "of" (count tx-chunks)
+                   "starting with block:" (pr-str (select-keys (first chunk) [:block/content :block/name])))
+          (d/transact! conn chunk)
+          (recur (rest chunks) (inc chunk-num)))))
+    #_(d/transact! conn blocks-tx)
+    (println "Created graph" (str db-name " with " (count (d/datoms @conn :eavt)) " datoms!"))))
+
+(when (= nbb/*file* (:file (meta #'-main)))
+  (-main *command-line-args*))

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -3,7 +3,7 @@
    Also creates a page of queries that exercises most properties
    NOTE: This script is also used in CI to confirm graph creation works"
   (:require [logseq.tasks.db-graph.create-graph :as create-graph]
-            [logseq.common.util :as common-util]
+            [logseq.common.util.date-time :as date-time-util]
             [logseq.db.frontend.property.type :as db-property-type]
             [clojure.string :as string]
             [datascript.core :as d]
@@ -12,15 +12,9 @@
             [nbb.core :as nbb]))
 
 (defn- date-journal-title [date]
-  (let [title (.toLocaleString date "en-US" #js {:month "short" :day "numeric" :year "numeric"})
-        suffixes {1 "st" 21 "st" 31 "st" 2 "nd" 22 "nd" 3 "rd" 23 "rd" 33 "rd"}]
-    (common-util/page-name-sanity-lc
-     (string/replace-first title #"(\d+)" (str "$1" (suffixes (.getDate date) "th"))))))
+  (string/lower-case (date-time-util/int->journal-title (date-time-util/date->int date) "MMM do, yyyy")))
 
-(defn- date-journal-day [date]
-  (js/parseInt (str (.toLocaleString date "en-US" #js {:year "numeric"})
-                    (.toLocaleString date "en-US" #js {:month "2-digit"})
-                    (.toLocaleString date "en-US" #js {:day "2-digit"}))))
+(def date-journal-day date-time-util/date->int)
 
 (defn- subtract-days
   [date days]

src/dev-cljs/shadow/build_large_graph.cljs

@@ -0,0 +1,59 @@
+(ns shadow.build-large-graph)
+
+(comment
+
+  (in-ns 'frontend.db-worker)
+  (def repo "logseq_db_large-db-demo")
+  (def conn (worker-state/get-datascript-conn repo))
+
+  (defonce *ids (atom (set (map :v (d/datoms @conn :avet :block/uuid)))))
+  (defn get-next-id
+    []
+    (let [id (random-uuid)]
+      (if (@*ids id)
+        (get-next-id)
+        (do
+          (swap! *ids conj id)
+          id))))
+
+  (defn pages
+    [start-idx n]
+    (let [ids (repeatedly n get-next-id)]
+      (map-indexed
+       (fn [idx id]
+         {:block/uuid id
+          :block/original-name (str "page-" (+ start-idx idx))
+          :block/name (str "page-" (+ start-idx idx))
+          :block/format :markdown})
+       ids)))
+
+  (defn blocks
+    [page-id size]
+    (let [page-id [:block/uuid page-id]
+          blocks (vec (repeatedly size (fn []
+                                         (let [id (get-next-id)]
+                                           {:block/uuid id
+                                            :block/content (str id)
+                                            :block/format :markdown
+                                            :block/page page-id
+                                            :block/parent page-id}))))]
+      (map-indexed
+       (fn [i b]
+         (if (zero? i)
+           (assoc b :block/left page-id)
+           (let [left (nth blocks (dec i))]
+             (assoc b :block/left [:block/uuid (:block/uuid left)]))))
+       blocks)))
+
+  (defn create-graph!
+    [conn page-size blocks-size start-idx]
+    (let [pages (pages start-idx page-size)
+          page-blocks (map (fn [p]
+                             (cons p
+                                   (blocks (:block/uuid p) blocks-size))) pages)]
+      (doseq [data (partition-all 1000 page-blocks)]
+        (let [tx-data (apply concat data)]
+          (prn :debug :progressing (:block/name (first tx-data)))
+          (d/transact! conn tx-data {:new-graph? true})))))
+
+  (create-graph! conn 30000 20 0))

src/electron/electron/handler.cljs

@@ -580,11 +580,6 @@
     (f window graph-name)
     (state/set-state! :window/once-graph-ready nil)))
 
-(defmethod handle :reloadWindowPage [^js win]
-  (logger/warn ::reload-window-page)
-  (when-let [web-content (.-webContents win)]
-    (.reload web-content)))
-
 (defmethod handle :window-minimize [^js win]
   (.minimize win))
 

src/main/frontend/components/block.cljs

@@ -82,9 +82,11 @@
             [promesa.core :as p]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
+            [frontend.rum :as r]
             [shadow.loader :as loader]
             [logseq.common.path :as path]
-            [electron.ipc :as ipc]))
+            [electron.ipc :as ipc]
+            [frontend.db.async :as db-async]))
 
 ;; local state
 (defonce *dragging?
@@ -509,14 +511,16 @@
   [e config page-name redirect-page-name page-name-in-block contents-page? whiteboard-page?]
   (util/stop e)
   (when (not (util/right-click? e))
-    (let [redirect-page-name (or redirect-page-name
-                                 (model/get-redirect-page-name page-name (:block/alias? config)))]
+    (p/let [redirect-page-name (or redirect-page-name
+                                   (model/get-redirect-page-name page-name (:block/alias? config)))
+            page (when redirect-page-name
+                   (db-async/<pull (state/get-current-repo) [:block/name (util/page-name-sanity-lc redirect-page-name)]))]
       (cond
         (gobj/get e "shiftKey")
-        (when-let [page-entity (db/entity [:block/name redirect-page-name])]
+        (when page
           (state/sidebar-add-block!
            (state/get-current-repo)
-           (:db/id page-entity)
+           (:db/id page)
            :page))
 
         (and (util/meta-key? e) (whiteboard-handler/inside-portal? (.-target e)))
@@ -527,11 +531,11 @@
         whiteboard-page?
         (route-handler/redirect-to-whiteboard! page-name)
 
-        (not= redirect-page-name page-name)
-        (route-handler/redirect-to-page! redirect-page-name)
+        (nil? page)
+        (state/pub-event! [:page/create page-name-in-block])
 
         :else
-        (state/pub-event! [:page/create page-name-in-block]))))
+        (route-handler/redirect-to-page! redirect-page-name))))
   (when (and contents-page?
              (util/mobile?)
              (state/get-left-sidebar-open?))
@@ -809,26 +813,34 @@
 (declare block-container)
 
 (rum/defc block-embed < rum/reactive
+  {:init (fn [state]
+           (let [block-id (second (:rum/args state))]
+             (db-async/<get-block (state/get-current-repo) block-id))
+           state)}
   [config uuid]
-  (when-let [block (db/entity [:block/uuid uuid])]
-    (let [repo (state/get-current-repo)]
-      (if (state/sub-block-unloaded? repo (str uuid))
-        [:span "Loading..."]
-        [:div.color-level.embed-block.bg-base-2
-         {:style {:z-index 2}
-          :on-double-click #(edit-parent-block % config)
-          :on-mouse-down (fn [e] (.stopPropagation e))}
-         [:div.px-3.pt-1.pb-2
-          (let [config' (assoc config
-                               :db/id (:db/id block)
-                               :id (str uuid)
-                               :embed-id uuid
-                               :embed? true
-                               :embed-parent (:block config)
-                               :ref? false)]
-            (blocks-container [block] config'))]]))))
+  (if (state/sub-async-query-loading (str uuid))
+    [:span "Loading..."]
+    (when-let [block (db/entity [:block/uuid uuid])]
+      [:div.color-level.embed-block.bg-base-2
+       {:style {:z-index 2}
+        :on-double-click #(edit-parent-block % config)
+        :on-mouse-down (fn [e] (.stopPropagation e))}
+       [:div.px-3.pt-1.pb-2
+        (let [config' (assoc config
+                             :db/id (:db/id block)
+                             :id (str uuid)
+                             :embed-id uuid
+                             :embed? true
+                             :embed-parent (:block config)
+                             :ref? false)]
+          (blocks-container [block] config'))]])))
 
 (rum/defc page-embed < rum/reactive db-mixins/query
+  {:init (fn [state]
+           (let [page-name (second (:rum/args state))
+                 page-name' (util/page-name-sanity-lc (string/trim page-name))]
+             (db-async/<get-block (state/get-current-repo) page-name'))
+           state)}
   [config page-name]
   (let [page-name (util/page-name-sanity-lc (string/trim page-name))
         current-page (state/get-current-page)
@@ -840,22 +852,24 @@
      [:section.flex.items-center.p-1.embed-header
       [:div.mr-3 svg/page]
       (page-cp config {:block/name page-name})]
-     (when (and
-            (not= (util/page-name-sanity-lc (or current-page ""))
-                  page-name)
-            (not= (util/page-name-sanity-lc (get config :id ""))
-                  page-name))
-       (if whiteboard-page?
-         ((state/get-component :whiteboard/tldraw-preview) page-name)
-         (let [block (model/get-page page-name)
-               block (db/sub-block (:db/id block))
-               blocks (db/sort-by-left (:block/_parent block) block)]
-           (blocks-container blocks (assoc config
-                                           :db/id (:db/id block)
-                                           :id page-name
-                                           :embed? true
-                                           :page-embed? true
-                                           :ref? false)))))]))
+     (if (and page-name (state/sub-async-query-loading page-name))
+       [:span "Loading..."]
+       (when (and
+              (not= (util/page-name-sanity-lc (or current-page ""))
+                    page-name)
+              (not= (util/page-name-sanity-lc (get config :id ""))
+                    page-name))
+         (if whiteboard-page?
+           ((state/get-component :whiteboard/tldraw-preview) page-name)
+           (let [block (model/get-page page-name)
+                 block (db/sub-block (:db/id block))
+                 blocks (db/sort-by-left (:block/_parent block) block)]
+             (blocks-container blocks (assoc config
+                                             :db/id (:db/id block)
+                                             :id page-name
+                                             :embed? true
+                                             :page-embed? true
+                                             :ref? false))))))]))
 
 (defn- get-label-text
   [label]
@@ -879,88 +893,91 @@
 (declare breadcrumb)
 
 (rum/defc block-reference < rum/reactive
+  {:init (fn [state]
+           (let [block-id (second (:rum/args state))]
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
+           state)}
   db-mixins/query
   [config id label]
   (if-let [block-id (if (uuid? id) id (parse-uuid id))]
-    (let [repo (state/get-current-repo)
-          block (db/entity [:block/uuid block-id])]
-      (if (state/sub-block-unloaded? repo (str block-id))
-        [:span "Loading..."]
-        (let [db-id (:db/id block)
-              block (when db-id (db/sub-block db-id))
-              properties (:block/properties block)
-              block-type (keyword (pu/lookup properties :ls-type))
-              hl-type (pu/lookup properties :hl-type)
-              repo (state/get-current-repo)
-              stop-inner-events? (= block-type :whiteboard-shape)]
-          (if (and block (:block/content block))
-            (let [title [:span.block-ref
-                         (block-content (assoc config :block-ref? true :stop-events? stop-inner-events?)
-                                        block nil (:block/uuid block)
-                                        (:slide? config)
-                                        false
-                                        (atom nil))]
-                  inner (if label
-                          (->elem
-                           :span.block-ref
-                           (map-inline config label))
-                          title)]
-              [:div.block-ref-wrap.inline
-               {:data-type    (name (or block-type :default))
-                :data-hl-type hl-type
-                :on-mouse-down
-                (fn [^js/MouseEvent e]
-                  (if (util/right-click? e)
-                    (state/set-state! :block-ref/context {:block (:block config)
-                                                          :block-ref block-id})
-                    (when (and
-                           (or (gobj/get e "shiftKey")
-                               (not (.. e -target (closest ".blank"))))
-                           (not (util/right-click? e)))
-                      (util/stop e)
-
-                      (cond
-                        (gobj/get e "shiftKey")
-                        (state/sidebar-add-block!
-                         (state/get-current-repo)
-                         (:db/id block)
-                         :block-ref)
-
-                        (and (util/meta-key? e) (whiteboard-handler/inside-portal? (.-target e)))
-                        (whiteboard-handler/add-new-block-portal-shape!
-                         (:block/uuid block)
-                         (whiteboard-handler/closest-shape (.-target e)))
-
-                        :else
-                        (match [block-type (util/electron?)]
+    (if (state/sub-async-query-loading (str block-id))
+      [:span "Loading..."]
+      (let [block (db/entity [:block/uuid block-id])
+            db-id (:db/id block)
+            block (when db-id (db/sub-block db-id))
+            properties (:block/properties block)
+            block-type (keyword (pu/lookup properties :ls-type))
+            hl-type (pu/lookup properties :hl-type)
+            repo (state/get-current-repo)
+            stop-inner-events? (= block-type :whiteboard-shape)]
+        (if (and block (:block/content block))
+          (let [title [:span.block-ref
+                       (block-content (assoc config :block-ref? true :stop-events? stop-inner-events?)
+                                      block nil (:block/uuid block)
+                                      (:slide? config)
+                                      false
+                                      (atom nil))]
+                inner (if label
+                        (->elem
+                         :span.block-ref
+                         (map-inline config label))
+                        title)]
+            [:div.block-ref-wrap.inline
+             {:data-type    (name (or block-type :default))
+              :data-hl-type hl-type
+              :on-mouse-down
+              (fn [^js/MouseEvent e]
+                (if (util/right-click? e)
+                  (state/set-state! :block-ref/context {:block (:block config)
+                                                        :block-ref block-id})
+                  (when (and
+                         (or (gobj/get e "shiftKey")
+                             (not (.. e -target (closest ".blank"))))
+                         (not (util/right-click? e)))
+                    (util/stop e)
+
+                    (cond
+                      (gobj/get e "shiftKey")
+                      (state/sidebar-add-block!
+                       (state/get-current-repo)
+                       (:db/id block)
+                       :block-ref)
+
+                      (and (util/meta-key? e) (whiteboard-handler/inside-portal? (.-target e)))
+                      (whiteboard-handler/add-new-block-portal-shape!
+                       (:block/uuid block)
+                       (whiteboard-handler/closest-shape (.-target e)))
+
+                      :else
+                      (match [block-type (util/electron?)]
                           ;; pdf annotation
-                          [:annotation true] (pdf-assets/open-block-ref! block)
+                        [:annotation true] (pdf-assets/open-block-ref! block)
 
-                          [:whiteboard-shape true] (route-handler/redirect-to-whiteboard!
-                                                    (get-in block [:block/page :block/name]) {:block-id block-id})
+                        [:whiteboard-shape true] (route-handler/redirect-to-whiteboard!
+                                                  (get-in block [:block/page :block/name]) {:block-id block-id})
 
                           ;; default open block page
-                          :else (route-handler/redirect-to-page! id))))))}
-
-               (if (and (not (util/mobile?))
-                        (not (:preview? config))
-                        (not (:modal/show? @state/state))
-                        (nil? block-type))
-                 (ui/tippy {:html        (fn []
-                                           [:div.tippy-wrapper.overflow-y-auto.p-4
-                                            {:style {:width      735
-                                                     :text-align "left"
-                                                     :max-height 600}}
-                                            [(breadcrumb config repo block-id {:indent? true})
-                                             (blocks-container
-                                              (db/get-block-and-children repo block-id)
-                                              (assoc config :id (str id) :preview? true))]])
-                            :interactive true
-                            :in-editor?  true
-                            :delay       [1000, 100]} inner)
-                 inner)])
-            [:span.warning.mr-1 {:title "Block ref invalid"}
-             (block-ref/->block-ref id)]))))
+                        :else (route-handler/redirect-to-page! id))))))}
+
+             (if (and (not (util/mobile?))
+                      (not (:preview? config))
+                      (not (:modal/show? @state/state))
+                      (nil? block-type))
+               (ui/tippy {:html        (fn []
+                                         [:div.tippy-wrapper.overflow-y-auto.p-4
+                                          {:style {:width      735
+                                                   :text-align "left"
+                                                   :max-height 600}}
+                                          [(breadcrumb config repo block-id {:indent? true})
+                                           (blocks-container
+                                            (db/get-block-and-children repo block-id)
+                                            (assoc config :id (str id) :preview? true))]])
+                          :interactive true
+                          :in-editor?  true
+                          :delay       [1000, 100]} inner)
+               inner)])
+          [:span.warning.mr-1 {:title "Block ref invalid"}
+           (block-ref/->block-ref id)])))
 
     [:span.warning.mr-1 {:title "Block ref invalid"}
      (block-ref/->block-ref id)]))
@@ -2487,10 +2504,16 @@
                  current-block-page? (= (str (:block/uuid block)) (state/get-current-page))
                  embed-self? (and (:embed? config)
                                   (= (:block/uuid block) (:block/uuid (:block config))))
-                 default-hide? (not (and current-block-page? (not embed-self?) (state/auto-expand-block-refs?)))]
-             (assoc state ::hide-block-refs? (atom default-hide?))))}
+                 default-hide? (not (and current-block-page? (not embed-self?) (state/auto-expand-block-refs?)))
+                 *refs-count (atom nil)]
+             (p/let [count (db-async/<get-block-refs-count (state/get-current-repo) (:db/id block))]
+               (reset! *refs-count count))
+             (assoc state
+                    ::hide-block-refs? (atom default-hide?)
+                    ::refs-count *refs-count)))}
   [state config {:block/keys [uuid format] :as block} edit-input-id block-id edit? hide-block-refs-count? selected? *ref]
   (let [*hide-block-refs? (get state ::hide-block-refs?)
+        *refs-count (get state ::refs-count)
         hide-block-refs? (rum/react *hide-block-refs?)
         editor-box (get config :editor-box)
         editor-id (str "editor-" edit-input-id)
@@ -2526,7 +2549,9 @@
              editor-cp
              (tags config block)]
             editor-cp))]
-       (let [refs-count (count (:block/_refs block))]
+       (let [refs-count (if (seq (:block/_refs block))
+                          (count (:block/_refs block))
+                          (rum/react *refs-count))]
          [:div.flex.flex-1.flex-col.block-content-wrapper
           [:div.flex.flex-row
            [:div.flex-1.w-full {:style {:display (if (:slide? config) "block" "flex")}}
@@ -2563,7 +2588,7 @@
               (block-refs-count block refs-count *hide-block-refs?)])]
 
           (when (and (not hide-block-refs?) (> refs-count 0))
-            (let [refs-cp (state/get-component :block/linked-references)]
+            (when-let [refs-cp (state/get-component :block/linked-references)]
               (refs-cp uuid)))]))]))
 
 (rum/defc single-block-cp
@@ -2623,21 +2648,30 @@
   (ui/icon "chevron-right" {:style {:font-size 20}
                             :class "opacity-50 mx-1"}))
 
-(defn breadcrumb
-  "block-id - uuid of the target block of breadcrumb. page uuid is also acceptable"
+;; "block-id - uuid of the target block of breadcrumb. page uuid is also acceptable"
+(rum/defc breadcrumb < rum/reactive
+  {:init (fn [state]
+           (let [args (:rum/args state)
+                 block-id (nth args 2)
+                 depth (:level-limit (last args))]
+             (p/let [id (:db/id (db/entity [:block/uuid block-id]))]
+               (when id (db-async/<get-block-parents (state/get-current-repo) id depth)))
+             state))}
   [config repo block-id {:keys [show-page? indent? end-separator? level-limit _navigating-block]
                          :or {show-page? true
                               level-limit 3}
                          :as opts}]
   (when block-id
-    (let [{:keys [from-block-id from-property-id]}
+    (let [_ (state/sub-async-query-loading (str block-id "-parents"))
+          {:keys [from-block-id from-property-id]}
           (when (and block-id (config/db-based-graph? repo))
             (db-property-handler/get-property-block-created-block [:block/uuid block-id]))
           from-block (when from-block-id (db/entity from-block-id))
           from-property (when from-property-id (db/entity from-property-id))
           block-id (or (:block/uuid from-block) block-id)
+          parents (db/get-block-parents repo block-id {:depth (inc level-limit)})
           parents (concat
-                   (db/get-block-parents repo block-id {:depth (inc level-limit)})
+                   parents
                    (when (and from-block from-property)
                      [from-block from-property]))
           page (or (db/get-block-page repo block-id) ;; only return for block uuid
@@ -2887,21 +2921,21 @@
 (defn- get-hidden-atom
   [sub-id *ref {:keys [initial-value]}]
   (let [*latest-value (atom nil)
-        *hidden? (rum/derived-atom [(:ui/main-container-scroll-top @state/state)] [::lazy-display sub-id]
-                   (fn [_top]
-                     (if (false? @*latest-value)
-                       @*latest-value
-                       (let [value (cond
-                                     (some? initial-value)
-                                     initial-value
-
-                                     @*ref
-                                     (hide-block? @*ref)
-
-                                     :else
-                                     true)]
-                         (reset! *latest-value value)
-                         value))))]
+        *hidden? (r/cached-derived-atom (:ui/main-container-scroll-top @state/state) [::lazy-display sub-id]
+                                        (fn [_top]
+                                          (if (false? @*latest-value)
+                                            @*latest-value
+                                            (let [value (cond
+                                                          (some? initial-value)
+                                                          initial-value
+
+                                                          @*ref
+                                                          (hide-block? @*ref)
+
+                                                          :else
+                                                          true)]
+                                              (reset! *latest-value value)
+                                              value))))]
     *hidden?))
 
 (rum/defcs ^:large-vars/cleanup-todo block-container-inner < rum/reactive db-mixins/query
@@ -2915,7 +2949,18 @@
                  *hidden? (get-hidden-atom id *ref
                                            {:initial-value (when (or disable-lazy? editing?) false)
                                             :id (:db/id current-block)
-                                            :content (:block/content current-block)})]
+                                            :content (:block/content current-block)})
+                 <load-block (fn []
+                               (let [block-id (:block/uuid (nth (:rum/args state) 3))]
+                                 (db-async/<get-block (state/get-current-repo) block-id :children? false)))]
+             (if (false? @*hidden?)
+               (<load-block)
+               (add-watch *hidden?
+                        :show
+                        (fn [_ _ _ n]
+                          (when (false? n)
+                            (<load-block)))))
+
              (assoc state
                     ::sub-id id
                     ::ref *ref
@@ -2947,6 +2992,11 @@
   (let [*ref (::ref state)
         ref (rum/react *ref)
         hidden? (rum/react (::hidden? state))
+        _ (when (:block/uuid block) (state/sub-async-query-loading (:block/uuid block)))
+        [original-block block] (build-block config* block {:navigating-block navigating-block :navigated? navigated?})
+        config* (if original-block
+                  (assoc config* :original-block original-block)
+                  config*)
         ref? (:ref? config*)
         ;; whiteboard block shape
         in-whiteboard? (and (:in-whiteboard? config*)
@@ -3049,8 +3099,8 @@
                            (block-mouse-leave e *control-show? block-id doc-mode?))}
         (when (and (not slide?) (not in-whiteboard?) (not hidden?))
           (let [edit? (or edit?
-                             (= uuid (:block/uuid (state/get-edit-block)))
-                             (contains? @(:editor/new-created-blocks @state/state) uuid))]
+                          (= uuid (:block/uuid (state/get-edit-block)))
+                          (contains? @(:editor/new-created-blocks @state/state) uuid))]
             (block-control config block uuid block-id collapsed? *control-show? edit? selected?)))
 
         (when (and @*show-left-menu? (not in-whiteboard?) (not hidden?))
@@ -3117,30 +3167,13 @@
                    state)}
   [state config block]
   (let [repo (state/get-current-repo)
-        unloaded? (state/sub-block-unloaded? repo (str (:block/uuid block)))
         *navigating-block (get state ::navigating-block)
         navigating-block (rum/react *navigating-block)
-        navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)
-        [original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?})
-        config' (if original-block
-                  (assoc config :original-block original-block)
-                  config)
-        opts {}]
-    (cond
-      unloaded?
-      [:div.ls-block.flex-1.flex-col.rounded-sm {:style {:width "100%"}}
-       [:div.flex.flex-row
-        [:div.flex.flex-row.items-center.mr-2.ml-1 {:style {:height 24}}
-         [:span.bullet-container.cursor
-          [:span.bullet]]]
-        [:div.flex.flex-1
-         [:span.opacity-70
-          "Loading..."]]]]
-
-      :else
+        navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)]
+    (when (:block/uuid block)
       (rum/with-key
-        (block-container-inner state repo config' block
-                               (merge opts {:navigating-block navigating-block :navigated? navigated?}))
+        (block-container-inner state repo config block
+                               {:navigating-block navigating-block :navigated? navigated?})
         (str "block-inner" (:block/uuid block))))))
 
 (defn divide-lists

src/main/frontend/components/cmdk.cljs

@@ -29,7 +29,8 @@
    [logseq.common.path :as path]
    [electron.ipc :as ipc]
    [frontend.util.text :as text-util]
-   [goog.userAgent]))
+   [goog.userAgent]
+   [frontend.db.async :as db-async]))
 
 (defn translate [t {:keys [id desc]}]
   (when id
@@ -414,20 +415,22 @@
     (state/close-modal!)))
 
 (defmethod handle-action :open-block [_ state _event]
-  (let [block-id (some-> state state->highlighted-item :source-block :block/uuid)
-        get-block-page (partial model/get-block-page (state/get-current-repo))
-        block (db/entity [:block/uuid block-id])]
-    (when block
-      (when-let [page (some-> block-id get-block-page)]
-        (let [page-name (:block/name page)]
-          (cond
-            (= (:block/type page) "whiteboard")
-            (route-handler/redirect-to-whiteboard! page-name {:block-id block-id})
-            (model/parents-collapsed? (state/get-current-repo) block-id)
-            (route-handler/redirect-to-page! (:block/uuid block))
-            :else
-            (route-handler/redirect-to-page! page-name {:anchor (str "ls-block-" block-id)})))
-        (state/close-modal!)))))
+  (when-let [block-id (some-> state state->highlighted-item :source-block :block/uuid)]
+    (p/let [repo (state/get-current-repo)
+            _ (db-async/<get-block repo block-id :children? false)]
+      (let [get-block-page (partial model/get-block-page repo)
+           block (db/entity [:block/uuid block-id])]
+       (when block
+         (when-let [page (some-> block-id get-block-page)]
+           (let [page-name (:block/name page)]
+             (cond
+               (= (:block/type page) "whiteboard")
+               (route-handler/redirect-to-whiteboard! page-name {:block-id block-id})
+               (model/parents-collapsed? (state/get-current-repo) block-id)
+               (route-handler/redirect-to-page! block-id)
+               :else
+               (route-handler/redirect-to-page! page-name {:anchor (str "ls-block-" block-id)})))
+           (state/close-modal!)))))))
 
 (defmethod handle-action :open-page-right [_ state _event]
   (when-let [page-name (get-highlighted-page-name state)]
@@ -439,8 +442,10 @@
 
 (defmethod handle-action :open-block-right [_ state _event]
   (when-let [block-uuid (some-> state state->highlighted-item :source-block :block/uuid)]
-    (editor-handler/open-block-in-sidebar! block-uuid)
-    (state/close-modal!)))
+    (p/let [repo (state/get-current-repo)
+            _ (db-async/<get-block repo block-uuid :children? false)]
+      (editor-handler/open-block-in-sidebar! block-uuid)
+      (state/close-modal!))))
 
 (defn- open-file
   [file-path]

src/main/frontend/components/container.cljs

@@ -603,9 +603,15 @@
          nil
 
          db-restoring?
-         [:div.mt-20
-          [:div.ls-center
-           (ui/loading)]]
+         (if config/publishing?
+           [:div.space-y-2
+            (shui/skeleton {:class "h-8 w-1/3 mb-8 bg-gray-400"})
+            (shui/skeleton {:class "h-6 w-full bg-gray-400"})
+            (shui/skeleton {:class "h-6 w-full bg-gray-400"})]
+           [:div.space-y-2
+            (shui/skeleton {:class "h-8 w-1/3 mb-8"})
+            (shui/skeleton {:class "h-6 w-full"})
+            (shui/skeleton {:class "h-6 w-full"})])
 
          :else
          [:div

src/main/frontend/components/file.cljs

@@ -20,43 +20,48 @@
             [promesa.core :as p]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
-            [logseq.common.path :as path]))
+            [logseq.common.path :as path]
+            [frontend.db.async :as db-async]))
 
 (defn- get-path
   [state]
   (let [route-match (first (:rum/args state))]
     (get-in route-match [:parameters :path :path])))
 
-(rum/defc files-all < rum/reactive
-  []
-  (when-let [current-repo (state/sub :git/current-repo)]
-    (let [files (db/get-files current-repo) ; [[string]]
-          files (sort-by first gstring/intAwareCompare files)
-          mobile? (util/mobile?)]
-      [:table.table-auto
-       [:thead
-        [:tr
-         [:th (t :file/name)]
-         (when-not mobile?
-           [:th (t :file/last-modified-at)])
-         (when-not mobile?
-           [:th ""])]]
-       [:tbody
-        (for [[file modified-at] files]
-          (let [file-id file]
-            [:tr {:key file-id}
-             [:td
-              (let [href (if (common-config/draw? file)
-                           (rfe/href :draw nil {:file (string/replace file (str common-config/default-draw-directory "/") "")})
-                           (rfe/href :file {:path file-id}))]
-                [:a {:href href}
-                 file])]
-             (when-not mobile?
-               [:td [:span.text-gray-500.text-sm
-                     (if (or (nil? modified-at) (zero? modified-at))
-                       (t :file/no-data)
-                       (date/get-date-time-string
-                        (t/to-default-time-zone (tc/to-date-time modified-at))))]])]))]])))
+(rum/defcs files-all < rum/reactive
+  {:init (fn [state]
+           (let [*files (atom nil)]
+             (p/let [result (db-async/<get-files (state/get-current-repo))]
+               (reset! *files result))
+             (assoc state ::files *files)))}
+  [state]
+  (let [files (rum/react (::files state))
+        files (sort-by first gstring/intAwareCompare files)
+        mobile? (util/mobile?)]
+    [:table.table-auto
+     [:thead
+      [:tr
+       [:th (t :file/name)]
+       (when-not mobile?
+         [:th (t :file/last-modified-at)])
+       (when-not mobile?
+         [:th ""])]]
+     [:tbody
+      (for [[file modified-at] files]
+        (let [file-id file]
+          [:tr {:key file-id}
+           [:td
+            (let [href (if (common-config/draw? file)
+                         (rfe/href :draw nil {:file (string/replace file (str common-config/default-draw-directory "/") "")})
+                         (rfe/href :file {:path file-id}))]
+              [:a {:href href}
+               file])]
+           (when-not mobile?
+             [:td [:span.text-gray-500.text-sm
+                   (if (or (nil? modified-at) (zero? modified-at))
+                     (t :file/no-data)
+                     (date/get-date-time-string
+                      (t/to-default-time-zone (tc/to-date-time modified-at))))]])]))]]))
 
 (rum/defc files
   []

src/main/frontend/components/imports.cljs

@@ -158,9 +158,9 @@
       {:auto-focus true
        :on-change (fn [e]
                     (reset! *input (util/evalue e)))
-       :on-key-press (fn [e]
-                       (when (= "Enter" (util/ekey e))
-                         (on-submit)))}]
+       :on-key-down (fn [e]
+                      (when (= "Enter" (util/ekey e))
+                        (on-submit)))}]
 
      [:div.mt-5.sm:mt-4.flex
       (ui/button "Submit"

src/main/frontend/components/page.cljs

@@ -23,7 +23,6 @@
             [frontend.db.model :as model]
             [frontend.extensions.graph :as graph]
             [frontend.extensions.pdf.utils :as pdf-utils]
-            [frontend.format.block :as block]
             [frontend.format.mldoc :as mldoc]
             [frontend.handler.common :as common-handler]
             [frontend.handler.config :as config-handler]
@@ -38,6 +37,7 @@
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.ui :as ui]
+            [logseq.shui.ui :as shui-ui]
             [frontend.util :as util]
             [frontend.util.text :as text-util]
             [goog.object :as gobj]
@@ -49,7 +49,8 @@
             [promesa.core :as p]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
-            [frontend.extensions.graph.pixi :as pixi]))
+            [frontend.extensions.graph.pixi :as pixi]
+            [frontend.db.async :as db-async]))
 
 (defn- get-page-name
   [state]
@@ -159,30 +160,26 @@
       (ui/icon "circle-plus")]]]])
 
 (rum/defcs page-blocks-cp < rum/reactive db-mixins/query
-                            {:will-mount (fn [state]
-                                           (let [page-e (second (:rum/args state))
-                                                 page-name (:block/name page-e)]
-                                             (when (and (db/journal-page? page-name)
-                                                     (>= (date/journal-title->int page-name)
-                                                       (date/journal-title->int (date/today))))
-                                               (state/pub-event! [:journal/insert-template page-name])))
-                                           state)}
-  [state repo page-e {:keys [sidebar? whiteboard?] :as config}]
+  {:will-mount (fn [state]
+                 (let [page-e (second (:rum/args state))
+                       page-name (:block/name page-e)]
+                   (when (and (db/journal-page? page-name)
+                              (>= (date/journal-title->int page-name)
+                                  (date/journal-title->int (date/today))))
+                     (state/pub-event! [:journal/insert-template page-name])))
+                 state)}
+  [state _repo page-e {:keys [sidebar? whiteboard?] :as config}]
   (when page-e
     (let [page-name (or (:block/name page-e)
-                      (str (:block/uuid page-e)))
+                        (str (:block/uuid page-e)))
           block-id (parse-uuid page-name)
           block? (boolean block-id)
           block (get-block page-name)
-          block-unloaded? (state/sub-block-unloaded? repo (:block/uuid block))
           children (:block/_parent block)]
       (cond
-        block-unloaded?
-        (ui/loading "Loading...")
-
         (and
-          (not block?)
-          (empty? children))
+         (not block?)
+         (empty? children))
         (dummy-block page-name)
 
         :else
@@ -207,11 +204,6 @@
                           {:page page-name})]
                (add-button args)))])))))
 
-(defn contents-page
-  [page]
-  (when-let [repo (state/get-current-repo)]
-    (page-blocks-cp repo page {:sidebar? true})))
-
 (rum/defc today-queries < rum/reactive
   [repo today? sidebar?]
   (when (and today? (not sidebar?))
@@ -446,140 +438,158 @@
     (state/get-current-page)))
 
 (defn- get-page-entity
-  [repo path-page-name page-name]
+  [page-name]
   (if-let [block-id (parse-uuid page-name)]
     (let [entity (db/entity [:block/uuid block-id])]
       entity)
-    (do
-      (when-not (db/entity repo [:block/name page-name])
-        (let [m (block/page-name->map path-page-name true)]
-          (db/transact! repo [m])))
-      (db/entity [:block/name page-name]))))
+    (db/entity [:block/name page-name])))
+
+(defn- get-sanity-page-name
+  [state page-name]
+  (when-let [path-page-name (get-path-page-name state page-name)]
+    (util/page-name-sanity-lc path-page-name)))
 
 ;; A page is just a logical block
 (rum/defcs ^:large-vars/cleanup-todo page-inner < rum/reactive db-mixins/query
-                                                  (rum/local false ::all-collapsed?)
-                                                  (rum/local false ::control-show?)
-                                                  (rum/local nil   ::current-page)
-                                                  (rum/local false ::hover-title?)
+  (rum/local false ::all-collapsed?)
+  (rum/local false ::control-show?)
+  (rum/local nil   ::current-page)
+  (rum/local false ::hover-title?)
+  {:init (fn [state]
+           (let [page-name (:page-name (first (:rum/args state)))
+                 page-name' (get-sanity-page-name state page-name)]
+             (db-async/<get-block (state/get-current-repo) page-name')
+             (assoc state ::page-name page-name')))}
   [state {:keys [repo page-name preview? sidebar?] :as option}]
-  (when-let [path-page-name (get-path-page-name state page-name)]
-    (let [current-repo (state/sub :git/current-repo)
-          repo (or repo current-repo)
-          page-name (util/page-name-sanity-lc path-page-name)
-          page (get-page-entity repo path-page-name page-name)
-          block-id (:block/uuid page)
-          block? (some? (:block/page page))
-          journal? (db/journal-page? page-name)
-          db-based? (config/db-based-graph? repo)
-          fmt-journal? (boolean (date/journal-title->int page-name))
-          whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape?
-          whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard?
-          route-page-name path-page-name
-          page-name (:block/name page)
-          page-original-name (:block/original-name page)
-          title (or page-original-name page-name)
-          today? (and
-                   journal?
-                   (= page-name (util/page-name-sanity-lc (date/journal-name))))
-          *control-show? (::control-show? state)
-          *all-collapsed? (::all-collapsed? state)
-          *current-block-page (::current-page state)
-          block-or-whiteboard? (or block? whiteboard?)
-          home? (= :home (state/get-current-route))]
-      (when (or page-name block-or-whiteboard?)
-        [:div.flex-1.page.relative
-         (merge (if (seq (:block/tags page))
-                  (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))]
-                    {:data-page-tags (text-util/build-data-value page-names)})
-                  {})
-
-           {:key path-page-name
-            :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])})
-
-         (if (and whiteboard-page? (not sidebar?))
-           [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive
-           [:div.relative
-            (when (and (not sidebar?) (not block?))
-              [:div.flex.flex-row.space-between
-               (when (or (mobile-util/native-platform?) (util/mobile?))
-                 [:div.flex.flex-row.pr-2
-                  {:style {:margin-left -15}
-                   :on-mouse-over (fn [e]
-                                    (page-mouse-over e *control-show? *all-collapsed?))
-                   :on-mouse-leave (fn [e]
-                                     (page-mouse-leave e *control-show?))}
-                  (page-blocks-collapse-control title *control-show? *all-collapsed?)])
-               (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))]
-                 (when (and (not whiteboard?) original-name)
-                   (page-title page-name {:journal? journal?
-                                          :fmt-journal? fmt-journal?
-                                          :preview? preview?
-                                          :*hover? (::hover-title? state)})))
-               (when (not config/publishing?)
-                 (when config/lsp-enabled?
-                   [:div.flex.flex-row
-                    (plugins/hook-ui-slot :page-head-actions-slotted nil)
-                    (plugins/hook-ui-items :pagebar)]))])
-
-            (when (and db-based? (not block?))
-              [:div.pb-4
-               (db-page/page-info page (::hover-title? state))])
-
-            [:div
-             (when (and block? (not sidebar?) (not whiteboard?))
-               (let [config {:id "block-parent"
-                             :block? true}]
-                 [:div.mb-4
-                  (component-block/breadcrumb config repo block-id {:level-limit 3})]))
-
-             (when (and db-based? (not block?) (not preview?))
-               (db-page/page-properties-react page {:configure? false}))
-
-             ;; blocks
-             (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page))))
-                   _ (when (and block? (not page))
-                       (route-handler/redirect-to-page! @*current-block-page))]
-               (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?}))]])
-
-         (when today?
-           (today-queries repo today? sidebar?))
-
-         (when today?
-           (scheduled/scheduled-and-deadlines page-name))
-
-         (when-not block?
-           (tagged-pages repo page-name page-original-name))
-
-         ;; referenced blocks
-         (when-not block-or-whiteboard?
-           [:div {:key "page-references"}
-            (rum/with-key
-              (reference/references route-page-name)
-              (str route-page-name "-refs"))])
-
-         (when-not block-or-whiteboard?
-           (when (not journal?)
-             (hierarchy/structures route-page-name)))
-
-         (when-not (or block-or-whiteboard? sidebar? home?)
-           [:div {:key "page-unlinked-references"}
-            (reference/unlinked-references route-page-name)])]))))
-
-(rum/defcs page < rum/reactive
+  (let [loading? (when (::page-name state)  (state/sub-async-query-loading (::page-name state)))]
+    (when-let [path-page-name (get-path-page-name state page-name)]
+      (let [current-repo (state/sub :git/current-repo)
+            repo (or repo current-repo)
+            page-name (util/page-name-sanity-lc path-page-name)
+            page (get-page-entity page-name)]
+        (when-not (and loading? (nil? page))
+          (let [block-id (:block/uuid page)
+                block? (some? (:block/page page))
+                journal? (db/journal-page? page-name)
+                db-based? (config/db-based-graph? repo)
+                fmt-journal? (boolean (date/journal-title->int page-name))
+                whiteboard? (:whiteboard? option) ;; in a whiteboard portal shape?
+                whiteboard-page? (model/whiteboard-page? page-name) ;; is this page a whiteboard?
+                route-page-name path-page-name
+                page-name (:block/name page)
+                page-original-name (:block/original-name page)
+                title (or page-original-name page-name)
+                today? (and
+                        journal?
+                        (= page-name (util/page-name-sanity-lc (date/journal-name))))
+                *control-show? (::control-show? state)
+                *all-collapsed? (::all-collapsed? state)
+                *current-block-page (::current-page state)
+                block-or-whiteboard? (or block? whiteboard?)
+                home? (= :home (state/get-current-route))]
+            (when (or page-name block-or-whiteboard?)
+              [:div.flex-1.page.relative
+               (merge (if (seq (:block/tags page))
+                        (let [page-names (model/get-page-names-by-ids (map :db/id (:block/tags page)))]
+                          {:data-page-tags (text-util/build-data-value page-names)})
+                        {})
+
+                      {:key path-page-name
+                       :class (util/classnames [{:is-journals (or journal? fmt-journal?)}])})
+
+               (if (and whiteboard-page? (not sidebar?))
+                 [:div ((state/get-component :whiteboard/tldraw-preview) page-name)] ;; FIXME: this is not reactive
+                 [:div.relative
+                  (when (and (not sidebar?) (not block?))
+                    [:div.flex.flex-row.space-between
+                     (when (or (mobile-util/native-platform?) (util/mobile?))
+                       [:div.flex.flex-row.pr-2
+                        {:style {:margin-left -15}
+                         :on-mouse-over (fn [e]
+                                          (page-mouse-over e *control-show? *all-collapsed?))
+                         :on-mouse-leave (fn [e]
+                                           (page-mouse-leave e *control-show?))}
+                        (page-blocks-collapse-control title *control-show? *all-collapsed?)])
+                     (let [original-name (:block/original-name (db/entity [:block/name (util/page-name-sanity-lc page-name)]))]
+                       (when (and (not whiteboard?) original-name)
+                         (page-title page-name {:journal? journal?
+                                                :fmt-journal? fmt-journal?
+                                                :preview? preview?
+                                                :*hover? (::hover-title? state)})))
+                     (when (not config/publishing?)
+                       (when config/lsp-enabled?
+                         [:div.flex.flex-row
+                          (plugins/hook-ui-slot :page-head-actions-slotted nil)
+                          (plugins/hook-ui-items :pagebar)]))])
+
+                  (cond
+                    (and db-based? (not block?))
+                    [:div.pb-4
+                     (db-page/page-info page (::hover-title? state))]
+
+                    (and (not db-based?) (not block?))
+                    [:div.pb-4])
+
+                  [:div
+                   (when (and block? (not sidebar?) (not whiteboard?))
+                     (let [config {:id "block-parent"
+                                   :block? true}]
+                       [:div.mb-4
+                        (component-block/breadcrumb config repo block-id {:level-limit 3})]))
+
+                   (when (and db-based? (not block?) (not preview?))
+                     (db-page/page-properties-react page {:configure? false}))
+
+                   ;; blocks
+                   (if loading?
+                     [:div.space-y-2
+                      (shui-ui/skeleton {:class "h-6 w-full"})
+                      (shui-ui/skeleton {:class "h-6 w-full"})]
+                     (let [_ (and block? page (reset! *current-block-page (:block/name (:block/page page))))
+                           _ (when (and block? (not page))
+                               (route-handler/redirect-to-page! @*current-block-page))]
+                       (page-blocks-cp repo page {:sidebar? sidebar? :whiteboard? whiteboard?})))]])
+
+               (when today?
+                 (today-queries repo today? sidebar?))
+
+               (when today?
+                 (scheduled/scheduled-and-deadlines page-name))
+
+               (when-not block?
+                 (tagged-pages repo page-name page-original-name))
+
+               ;; referenced blocks
+               (when-not block-or-whiteboard?
+                 (when page
+                   [:div {:key "page-references"}
+                    (rum/with-key
+                      (reference/references route-page-name)
+                      (str route-page-name "-refs"))]))
+
+               (when-not block-or-whiteboard?
+                 (when (not journal?)
+                   (hierarchy/structures route-page-name)))
+
+               (when-not (or block-or-whiteboard? sidebar? home?)
+                 [:div {:key "page-unlinked-references"}
+                  (reference/unlinked-references route-page-name)])])))))))
+
+(rum/defcs page < rum/static
   [state option]
-  (let [path-page-name (get-path-page-name state (:page-name option))
-        page-name (util/page-name-sanity-lc path-page-name)
-        repo (state/get-current-repo)
-        page (get-page-entity repo path-page-name page-name)
-        block? (some? (:block/page page))
-        page-unloaded? (or (state/sub-page-unloaded? repo page-name) (nil? page))]
-    (if (and page-unloaded? (not block?))
-      (state/update-state! [repo :unloaded-pages] (fn [pages] (conj (set pages) page-name)))
-      (rum/with-key
-        (page-inner option)
-        (or (:page-name option)
-          (get-page-name state))))))
+  (rum/with-key
+    (page-inner option)
+    (or (:page-name option)
+        (get-page-name state))))
+
+(rum/defc contents-page < rum/reactive
+  {:init (fn [state]
+           (db-async/<get-block (state/get-current-repo) "contents")
+           state)}
+  [page]
+  (when-let [repo (state/get-current-repo)]
+    (when-not (state/sub-async-query-loading "contents")
+      (page-blocks-cp repo page {:sidebar? true}))))
 
 (defonce layout (atom [js/window.innerWidth js/window.innerHeight]))
 
@@ -1043,21 +1053,21 @@
         :on-click close-fn)
 
       (ui/button
-        (t :yes)
-        :on-click (fn []
-                    (close-fn)
-                    (let [failed-pages (atom [])]
-                      (doseq [page-name (map :block/name pages)]
-                        (page-handler/delete! page-name #()
-                          {:error-handler
-                           (fn [msg]
-                             (js/console.error msg)
-                             (swap! failed-pages conj page-name))}))
-                      (if (seq @failed-pages)
-                        (notification/show! (t :all-pages/failed-to-delete-pages (string/join ", " (map pr-str @failed-pages)))
-                          :warning false)
-                        (notification/show! (t :tips/all-done) :success)))
-                    (js/setTimeout #(refresh-fn) 200)))]]))
+       (t :yes)
+       :on-click (fn []
+                   (close-fn)
+                   (let [failed-pages (atom [])]
+                     (p/let [_ (p/all (map (fn [page-name]
+                                             (page-handler/<delete! page-name nil
+                                                                    {:error-handler
+                                                                     (fn [msg]
+                                                                       (js/console.error msg)
+                                                                       (swap! failed-pages conj page-name))})) (map :block/name pages)))]
+                       (if (seq @failed-pages)
+                         (notification/show! (t :all-pages/failed-to-delete-pages (string/join ", " (map pr-str @failed-pages)))
+                                             :warning false)
+                         (notification/show! (t :tips/all-done) :success))))
+                   (js/setTimeout #(refresh-fn) 200)))]]))
 
 (rum/defc pagination
   "Pagination component, like `<< <Prev 1/10 Next> >>`.
@@ -1115,6 +1125,7 @@
         *search-key (::search-key state)
         *search-input (rum/create-ref)
 
+        ;; TODO: remove this
         *indeterminate (rum/derived-atom
                          [*checks] ::indeterminate
                          (fn [checks]

src/main/frontend/components/page.css

@@ -231,13 +231,17 @@
 .ls-page-title {
   @apply rounded-sm;
 
-  padding: 5px 8px;
+  padding: 5px 8px 12px 8px;
   margin: 0 -6px;
 
   &.title {
     margin-bottom: 12px;
   }
 
+  h1.page-title {
+    margin-bottom: 0;
+  }
+
   .edit-input {
     @apply w-full border-0 p-0 pr-1 bg-transparent outline-0;
 
@@ -251,6 +255,10 @@
       }
     }
   }
+
+  .page-icon {
+      font-size: 48px;
+  }
 }
 
 a.page-title {
@@ -378,3 +386,12 @@ html.is-native-ios {
 .references {
   user-select: none;
 }
+
+.page-info {
+    min-height: 46px;
+    margin-left: -21px;
+}
+
+.page-info-title-placeholder {
+    min-height: 28px;
+}

src/main/frontend/components/page_menu.cljs

@@ -22,7 +22,7 @@
 
 (defn- delete-page!
   [page-name]
-  (page-handler/delete! page-name
+  (page-handler/<delete! page-name
                         (fn []
                           (notification/show! (str "Page " page-name " was deleted successfully!")
                                               :success))

src/main/frontend/components/property/value.cljs

@@ -20,7 +20,8 @@
             [rum.core :as rum]
             [frontend.handler.route :as route-handler]
             [frontend.handler.property.util :as pu]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.db.async :as db-async]))
 
 (defn- select-type?
   [property type]
@@ -443,9 +444,13 @@
                            :editor-box editor-box})])))
 
 (rum/defc property-template-value < rum/reactive
+  {:init (fn [state]
+           (let [block-id (second (:rum/args state))]
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
+           state)}
   [config value opts]
   (when value
-    (if (state/sub-block-unloaded? (state/get-current-repo) value)
+    (if (state/sub-async-query-loading value)
       [:div.text-sm.opacity-70 "loading"]
       (when-let [entity (db/sub-block (:db/id (db/entity [:block/uuid value])))]
         (let [properties-cp (:properties-cp opts)]
@@ -461,11 +466,15 @@
 
 (rum/defcs property-block-value < rum/reactive
   (rum/local nil ::template-instance)
+  {:init (fn [state]
+           (let [block-id (first (:rum/args state))]
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
+           state)}
   [state value block property block-cp editor-box opts page-cp editor-id]
   (let [*template-instance (::template-instance state)
         template-instance @*template-instance]
     (when value
-      (if (state/sub-block-unloaded? (state/get-current-repo) value)
+      (if (state/sub-async-query-loading value)
         [:div.text-sm.opacity-70 "loading"]
         (when-let [v-block (db/sub-block (:db/id (db/entity [:block/uuid value])))]
           (let [class? (contains? (:block/type v-block) "class")
@@ -493,9 +502,13 @@
               invalid-warning)))))))
 
 (rum/defc closed-value-item < rum/reactive
+  {:init (fn [state]
+           (let [block-id (first (:rum/args state))]
+             (db-async/<get-block (state/get-current-repo) block-id :children? false))
+           state)}
   [value {:keys [page-cp inline-text icon?]}]
   (when value
-    (if (state/sub-block-unloaded? (state/get-current-repo) value)
+    (if (state/sub-async-query-loading value)
       [:div.text-sm.opacity-70 "loading"]
       (when-let [block (db/sub-block (:db/id (db/entity [:block/uuid value])))]
         (let [value' (get-in block [:block/schema :value])

src/main/frontend/components/reference.cljs

@@ -16,7 +16,9 @@
             [frontend.ui :as ui]
             [frontend.util :as util]
             [rum.core :as rum]
-            [frontend.modules.outliner.tree :as tree]))
+            [frontend.modules.outliner.tree :as tree]
+            [frontend.db.async :as db-async]
+            [promesa.core :as p]))
 
 (defn- frequencies-sort
   [references]
@@ -96,23 +98,29 @@
     (filter-dialog-inner filters-atom *references page-name)))
 
 (rum/defc block-linked-references < rum/reactive db-mixins/query
+  {:init (fn [state]
+           (when-let [e (db/entity [:block/uuid (first (:rum/args state))])]
+             (db-async/<get-block-refs (state/get-current-repo) (:db/id e)))
+           state)}
   [block-id]
-  (let [e (db/entity [:block/uuid block-id])
-        page? (some? (:block/name e))
-        ref-blocks (if page?
-                     (-> (db/get-page-referenced-blocks (:block/name e))
-                         db-utils/group-by-page)
-                     (db/get-block-referenced-blocks block-id))
-        ref-hiccup (block/->hiccup ref-blocks
-                                   {:id (str block-id)
-                                    :ref? true
-                                    :breadcrumb-show? true
-                                    :group-by-page? true
-                                    :editor-box editor/box}
-                                   {})]
-    [:div.references-blocks
-     (content/content block-id
-                      {:hiccup ref-hiccup})]))
+  (when-let [e (db/entity [:block/uuid block-id])]
+    (when-not (state/sub-async-query-loading (str (:db/id e) "-refs"))
+      (let [page? (some? (:block/name e))
+            ref-blocks (if page?
+                         (-> (db/get-page-referenced-blocks (:block/name e))
+                             db-utils/group-by-page)
+                         (db/get-block-referenced-blocks block-id))]
+        (when (> (count ref-blocks) 0)
+          (let [ref-hiccup (block/->hiccup ref-blocks
+                                           {:id (str block-id)
+                                            :ref? true
+                                            :breadcrumb-show? true
+                                            :group-by-page? true
+                                            :editor-box editor/box}
+                                           {})]
+            [:div.references-blocks
+             (content/content block-id
+                              {:hiccup ref-hiccup})]))))))
 
 (rum/defc references-inner
   [page-name filters filtered-ref-blocks]
@@ -191,59 +199,65 @@
 (rum/defcs references* < rum/reactive db-mixins/query
   (rum/local nil ::ref-pages)
   {:init (fn [state]
-           (let [page-name (first (:rum/args state))
+           (let [page-name (->> (first (:rum/args state))
+                                util/page-name-sanity-lc)
+                 page (db/entity [:block/name page-name])
                  filters (when page-name (atom nil))]
+             (when page (db-async/<get-block-refs (state/get-current-repo) (:db/id page)))
              (assoc state ::filters filters)))}
   [state page-name]
   (when page-name
-    (let [page-name (util/page-name-sanity-lc page-name)
-          page-props-v (state/sub-page-properties-changed page-name)
-          *ref-pages (::ref-pages state)
-          repo (state/get-current-repo)
-          filters-atom (get state ::filters)
-          filter-state (rum/react filters-atom)
-          ref-blocks (db/get-page-referenced-blocks page-name)
-          page-id (:db/id (db/entity repo [:block/name page-name]))
-          aliases (db/page-alias-set repo page-name)
-          aliases-exclude-self (set (remove #{page-id} aliases))
-          top-level-blocks (filter (fn [b] (some aliases (set (map :db/id (:block/refs b))))) ref-blocks)
-          top-level-blocks-ids (set (map :db/id top-level-blocks))
-          filters (when (seq filter-state)
-                    (-> (group-by second filter-state)
-                        (update-vals #(map first %))))
-          filtered-ref-blocks (->> (block-handler/filter-blocks ref-blocks filters)
-                                   (block-handler/get-filtered-ref-blocks-with-parents ref-blocks))
-          total (count top-level-blocks)
-          filtered-top-blocks (filter (fn [b] (top-level-blocks-ids (:db/id b))) filtered-ref-blocks)
-          filter-n (count filtered-top-blocks)
-          parent->blocks (group-by (fn [x] (:db/id (x :block/parent))) filtered-ref-blocks)
-          result (->> (group-by :block/page filtered-top-blocks)
-                      (map (fn [[page blocks]]
-                             (let [blocks (sort-by (fn [b] (not= (:db/id page) (:db/id (:block/parent b)))) blocks)
-                                   result (map (fn [block]
-                                                 (let [filtered-children (get-filtered-children block parent->blocks)
-                                                       refs (when-not (contains? top-level-blocks-ids (:db/id (:block/parent block)))
-                                                              (block-handler/get-blocks-refed-pages aliases (cons block filtered-children)))
-                                                       block' (assoc (tree/block-entity->map block) :block/children filtered-children)]
-                                                   [block' refs])) blocks)
-                                   blocks' (map first result)
-                                   page' (if (contains? aliases-exclude-self (:db/id page))
-                                           {:db/id (:db/id page)
-                                            :block/alias? true
-                                            :block/journal-day (:block/journal-day page)}
-                                           page)]
-                               [[page' blocks'] (mapcat second result)]))))
-          filtered-ref-blocks' (map first result)
-          ref-pages (->>
-                     (mapcat second result)
-                     (map :block/original-name)
-                     frequencies)]
-      (reset! *ref-pages ref-pages)
-      (when (or (seq filter-state) (> filter-n 0))
-        [:div.references.page-linked.flex-1.flex-row
-         (sub-page-properties-changed page-name page-props-v filters-atom)
-         [:div.content.pt-6
-          (references-cp page-name filters filters-atom filter-state total filter-n filtered-ref-blocks' *ref-pages)]]))))
+    (let [repo (state/get-current-repo)
+          page-name (util/page-name-sanity-lc page-name)
+          page-entity (db/entity repo [:block/name page-name])]
+      (when page-entity
+        (when-not (state/sub-async-query-loading (str (:db/id page-entity) "-refs"))
+          (let [page-props-v (state/sub-page-properties-changed page-name)
+                *ref-pages (::ref-pages state)
+                filters-atom (get state ::filters)
+                filter-state (rum/react filters-atom)
+                ref-blocks (db/get-page-referenced-blocks page-name)
+                page-id (:db/id page-entity)
+                aliases (db/page-alias-set repo page-name)
+                aliases-exclude-self (set (remove #{page-id} aliases))
+                top-level-blocks (filter (fn [b] (some aliases (set (map :db/id (:block/refs b))))) ref-blocks)
+                top-level-blocks-ids (set (map :db/id top-level-blocks))
+                filters (when (seq filter-state)
+                          (-> (group-by second filter-state)
+                              (update-vals #(map first %))))
+                filtered-ref-blocks (->> (block-handler/filter-blocks ref-blocks filters)
+                                         (block-handler/get-filtered-ref-blocks-with-parents ref-blocks))
+                total (count top-level-blocks)
+                filtered-top-blocks (filter (fn [b] (top-level-blocks-ids (:db/id b))) filtered-ref-blocks)
+                filter-n (count filtered-top-blocks)
+                parent->blocks (group-by (fn [x] (:db/id (x :block/parent))) filtered-ref-blocks)
+                result (->> (group-by :block/page filtered-top-blocks)
+                            (map (fn [[page blocks]]
+                                   (let [blocks (sort-by (fn [b] (not= (:db/id page) (:db/id (:block/parent b)))) blocks)
+                                         result (map (fn [block]
+                                                       (let [filtered-children (get-filtered-children block parent->blocks)
+                                                             refs (when-not (contains? top-level-blocks-ids (:db/id (:block/parent block)))
+                                                                    (block-handler/get-blocks-refed-pages aliases (cons block filtered-children)))
+                                                             block' (assoc (tree/block-entity->map block) :block/children filtered-children)]
+                                                         [block' refs])) blocks)
+                                         blocks' (map first result)
+                                         page' (if (contains? aliases-exclude-self (:db/id page))
+                                                 {:db/id (:db/id page)
+                                                  :block/alias? true
+                                                  :block/journal-day (:block/journal-day page)}
+                                                 page)]
+                                     [[page' blocks'] (mapcat second result)]))))
+                filtered-ref-blocks' (map first result)
+                ref-pages (->>
+                           (mapcat second result)
+                           (map :block/original-name)
+                           frequencies)]
+            (reset! *ref-pages ref-pages)
+            (when (or (seq filter-state) (> filter-n 0))
+              [:div.references.page-linked.flex-1.flex-row
+               (sub-page-properties-changed page-name page-props-v filters-atom)
+               [:div.content.pt-6
+                (references-cp page-name filters filters-atom filter-state total filter-n filtered-ref-blocks' *ref-pages)]])))))))
 
 (rum/defc references
   [page-name]
@@ -258,27 +272,26 @@
 
 (rum/defcs unlinked-references-aux
   < rum/reactive db-mixins/query
-  {:wrap-render
-   (fn [render-fn]
-     (fn [state]
-       (reset! (second (:rum/args state))
-               (apply +
-                      (for [[_ rfs]
-                            (db/get-page-unlinked-references
-                             (first (:rum/args state)))]
-                        (count rfs))))
-       (render-fn state)))}
+  {:init
+   (fn [state]
+     (let [*result (atom nil)
+           [page-name *n-ref] (:rum/args state)]
+       (p/let [result (search/get-page-unlinked-refs page-name)]
+         (reset! *n-ref (count result))
+         (reset! *result result))
+       (assoc state ::result *result)))}
   [state page-name _n-ref]
-  (let [ref-blocks (db/get-page-unlinked-references page-name)]
-    [:div.references-blocks
-     (let [ref-hiccup (block/->hiccup ref-blocks
-                                      {:id (str page-name "-unlinked-")
-                                       :ref? true
-                                       :group-by-page? true
-                                       :editor-box editor/box}
-                                      {})]
-       (content/content page-name
-                        {:hiccup ref-hiccup}))]))
+  (let [ref-blocks (rum/react (::result state))]
+    (when (seq ref-blocks)
+      [:div.references-blocks
+       (let [ref-hiccup (block/->hiccup ref-blocks
+                                        {:id (str page-name "-unlinked-")
+                                         :ref? true
+                                         :group-by-page? true
+                                         :editor-box editor/box}
+                                        {})]
+         (content/content page-name
+                          {:hiccup ref-hiccup}))])))
 
 (rum/defcs unlinked-references < rum/reactive
   (rum/local nil ::n-ref)

src/main/frontend/components/scheduled_deadlines.cljs

@@ -7,8 +7,8 @@
             [clojure.string :as string]
             [frontend.components.editor :as editor]
             [rum.core :as rum]
-            [frontend.db :as db]
-            [frontend.db-mixins :as db-mixins]))
+            [frontend.db.async :as db-async]
+            [promesa.core :as p]))
 
 (defn- scheduled-or-deadlines?
   [page-name]
@@ -16,10 +16,16 @@
        (not (true? (state/scheduled-deadlines-disabled?)))
        (= (string/lower-case page-name) (string/lower-case (date/journal-name)))))
 
-(rum/defc scheduled-and-deadlines-inner < rum/reactive db-mixins/query
-  [page-name]
-  (let [scheduled-or-deadlines (when (scheduled-or-deadlines? page-name)
-                                 (db/get-date-scheduled-or-deadlines (string/capitalize page-name)))]
+(rum/defcs scheduled-and-deadlines-inner < rum/reactive
+  {:init (fn [state]
+           (let [*result (atom nil)
+                 page-name (first (:rum/args state))]
+             (p/let [result (when (scheduled-or-deadlines? page-name)
+                              (db-async/<get-date-scheduled-or-deadlines (string/capitalize page-name)))]
+               (reset! *result result))
+             (assoc state ::result *result)))}
+  [state page-name]
+  (let [scheduled-or-deadlines (rum/react (::result state))]
     (when (seq scheduled-or-deadlines)
       [:div.scheduled-or-deadlines.mt-8
        (ui/foldable

src/main/frontend/components/whiteboard.cljs

@@ -20,7 +20,8 @@
             [promesa.core :as p]
             [rum.core :as rum]
             [shadow.loader :as loader]
-            [frontend.config :as config]))
+            [frontend.config :as config]
+            [frontend.db.async :as db-async]))
 
 (defonce tldraw-loaded? (atom false))
 (rum/defc tldraw-app < rum/reactive
@@ -39,13 +40,15 @@
   {:init (fn [state]
            (p/let [_ (loader/load :tldraw)]
              (reset! tldraw-loaded? true))
+           (let [page-name (first (:rum/args state))]
+             (db-async/<get-block (state/get-current-repo) page-name))
            state)}
   [page-name]
   (let [loaded? (rum/react tldraw-loaded?)
         tldr (whiteboard-handler/page-name->tldr! page-name)
         generate-preview (when loaded?
                            (resolve 'frontend.extensions.tldraw/generate-preview))]
-    (when generate-preview
+    (when (and generate-preview (not (state/sub-async-query-loading page-name)))
       (generate-preview tldr))))
 
 ;; TODO: use frontend.ui instead of making a new one

src/main/frontend/date.cljs

@@ -7,7 +7,7 @@
             [cljs-time.format :as tf]
             [cljs-time.local :as tl]
             [frontend.state :as state]
-            [logseq.graph-parser.date-time-util :as date-time-util]
+            [logseq.common.util.date-time :as date-time-util]
             [goog.object :as gobj]
             [lambdaisland.glogi :as log]
             [frontend.worker.date :as worker-date]))

src/main/frontend/db.cljs

@@ -22,7 +22,7 @@
   remove-conn!]
 
  [frontend.db.utils
-  db->json db->edn-str db->string get-max-tx-id get-tx-id
+  db->edn-str db->string get-max-tx-id get-tx-id
   group-by-page seq-flatten
   string->db
 
@@ -32,14 +32,14 @@
   delete-blocks get-pre-block
   delete-files delete-pages-by-files get-all-tagged-pages
   get-block-and-children get-block-by-uuid get-block-children sort-by-left
-  get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
+  get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks
   get-block-immediate-children get-block-page
-  get-custom-css get-date-scheduled-or-deadlines
+  get-custom-css
   get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
-  get-files get-files-blocks get-files-full get-journals-length get-pages-with-file
+  get-files-blocks get-files-full get-journals-length get-pages-with-file
   get-latest-journals get-page get-page-alias get-page-alias-names
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
-  get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages get-page-unlinked-references
+  get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages
   get-all-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages
   journal-page? page-alias-set sub-block
   set-file-last-modified-at! page-empty? page-exists? page-empty-or-dummy? get-alias-source-page

src/main/frontend/db/async.cljs

@@ -8,9 +8,21 @@
             [frontend.util :as util]
             [frontend.db.utils :as db-utils]
             [frontend.db.async.util :as db-async-util]
-            [frontend.db.file-based.async :as file-async]))
+            [frontend.db.file-based.async :as file-async]
+            [frontend.db :as db]
+            [frontend.db.model :as db-model]
+            [frontend.persist-db.browser :as db-browser]
+            [clojure.edn :as edn]
+            [datascript.core :as d]
+            [frontend.db.react :as react]
+            [frontend.date :as date]
+            [cljs-time.core :as t]
+            [cljs-time.format :as tf]))
 
 (def <q db-async-util/<q)
+(def <pull db-async-util/<pull)
+(comment
+  (def <pull-many db-async-util/<pull-many))
 
 (defn <get-files
   [graph]
@@ -25,12 +37,18 @@
 (defn <get-all-templates
   [graph]
   (p/let [result (<q graph
-                     '[:find ?t ?b
+                     '[:find ?t (pull ?b [*])
                        :where
                        [?b :block/properties ?p]
                        [(get ?p :template) ?t]])]
     (into {} result)))
 
+(defn <get-template-by-name
+  [name]
+  (let [repo (state/get-current-repo)]
+    (p/let [templates (<get-all-templates repo)]
+      (get templates name))))
+
 (defn <db-based-get-all-properties
   ":block/type could be one of [property, class]."
   [graph]
@@ -98,3 +116,128 @@
   (if (config/db-based-graph? graph)
     (<get-db-based-property-values graph property)
     (file-async/<get-file-based-property-values graph property)))
+
+;; TODO: batch queries for better performance and UX
+(defn <get-block
+  [graph name-or-uuid & {:keys [children?]
+                         :or {children? true}}]
+  (let [name' (str name-or-uuid)
+        e (cond
+            (number? name-or-uuid)
+            (db/entity name-or-uuid)
+            (util/uuid-string? name')
+            (db/entity [:block/uuid (uuid name')])
+            :else
+            (db/entity [:block/name (util/page-name-sanity-lc name')]))]
+    (if (:block.temp/fully-loaded? e)
+      e
+      (when-let [^Object sqlite @db-browser/*worker]
+        (state/update-state! :db/async-queries (fn [s] (conj s name')))
+        (p/let [result (.get-block-and-children sqlite graph name' children?)
+                {:keys [block children] :as result'} (edn/read-string result)
+                conn (db/get-db graph false)
+                block-and-children (cons block children)
+                _ (d/transact! conn block-and-children)]
+          (state/update-state! :db/async-queries (fn [s] (disj s name')))
+          (react/refresh-affected-queries!
+           graph
+           [[:frontend.worker.react/block (:db/id block)]])
+          (if children?
+            block
+            result'))))))
+
+(defn <get-right-sibling
+  [graph db-id]
+  (assert (integer? db-id))
+  (when-let [^Object worker @db-browser/*worker]
+    (p/let [result-str (.get-right-sibling worker graph db-id)
+            result (edn/read-string result-str)
+            conn (db/get-db graph false)
+            _ (when result (d/transact! conn [result]))]
+      result)))
+
+(defn <get-block-parents
+  [graph id depth]
+  (assert (integer? id))
+  (when-let [^Object worker @db-browser/*worker]
+    (when-let [block-id (:block/uuid (db/entity graph id))]
+      (state/update-state! :db/async-queries (fn [s] (conj s (str block-id "-parents"))))
+      (p/let [result-str (.get-block-parents worker graph id depth)
+              result (edn/read-string result-str)
+              conn (db/get-db graph false)
+              _ (d/transact! conn result)]
+        (state/update-state! :db/async-queries (fn [s] (disj s (str block-id "-parents"))))
+        result))))
+
+(defn <get-block-refs
+  [graph eid]
+  (assert (integer? eid))
+  (when-let [^Object worker @db-browser/*worker]
+    (state/update-state! :db/async-queries (fn [s] (conj s (str eid "-refs"))))
+    (p/let [result-str (.get-block-refs worker graph eid)
+            result (edn/read-string result-str)
+            conn (db/get-db graph false)
+            _ (d/transact! conn result)]
+      (state/update-state! :db/async-queries (fn [s] (disj s (str eid "-refs"))))
+      result)))
+
+(defn <get-block-refs-count
+  [graph eid]
+  (assert (integer? eid))
+  (when-let [^Object worker @db-browser/*worker]
+    (.get-block-refs-count worker graph eid)))
+
+(defn <get-all-referenced-blocks-uuid
+  "Get all uuids of blocks with any back link exists."
+  [graph]
+  (<q graph
+      '[:find [?refed-uuid ...]
+        :where
+           ;; ?referee-b is block with ref towards ?refed-b
+        [?refed-b   :block/uuid ?refed-uuid]
+        [?referee-b :block/refs ?refed-b]]))
+
+(defn <get-file
+  [graph path]
+  (when (and graph path)
+    (p/let [result (<pull graph [:file/path path])]
+      (:file/content result))))
+
+(defn <get-date-scheduled-or-deadlines
+  [journal-title]
+  (when-let [date (date/journal-title->int journal-title)]
+    (let [future-days (state/get-scheduled-future-days)
+          date-format (tf/formatter "yyyyMMdd")
+          current-day (tf/parse date-format (str date))
+          future-day (some->> (t/plus current-day (t/days future-days))
+                              (tf/unparse date-format)
+                              (parse-long))]
+      (when future-day
+        (when-let [repo (state/get-current-repo)]
+          (p/let [result (<q repo
+                             '[:find [(pull ?block ?block-attrs) ...]
+                               :in $ ?day ?future ?block-attrs
+                               :where
+                               (or
+                                [?block :block/scheduled ?d]
+                                [?block :block/deadline ?d])
+                               [(get-else $ ?block :block/repeated? false) ?repeated]
+                               [(get-else $ ?block :block/marker "NIL") ?marker]
+                               [(not= ?marker "DONE")]
+                               [(not= ?marker "CANCELED")]
+                               [(not= ?marker "CANCELLED")]
+                               [(<= ?d ?future)]
+                               (or-join [?repeated ?d ?day]
+                                        [(true? ?repeated)]
+                                        [(>= ?d ?day)])]
+                             date
+                             future-day
+                             db-model/block-attrs)]
+            (->> result
+                 db-model/sort-by-left-recursive
+                 db-utils/group-by-page)))))))
+
+(defn <fetch-all-pages
+  [graph]
+  (when-let [^Object worker @db-browser/*worker]
+    (.fetch-all-pages worker graph)))

src/main/frontend/db/async/util.cljs

@@ -1,12 +1,49 @@
 (ns frontend.db.async.util
   "Async util helper"
-  (:require [frontend.persist-db.browser :as db-browser]
-            [cljs-bean.core :as bean]
-            [promesa.core :as p]))
+  (:require [frontend.state :as state]
+            [promesa.core :as p]
+            [clojure.edn :as edn]
+            [frontend.db.conn :as db-conn]
+            [datascript.core :as d]))
 
 (defn <q
   [graph & inputs]
   (assert (not-any? fn? inputs) "Async query inputs can't include fns because fn can't be serialized")
-  (when-let [sqlite @db-browser/*worker]
+  (when-let [^Object sqlite @state/*db-worker]
     (p/let [result (.q sqlite graph (pr-str inputs))]
-      (bean/->clj result))))
+      (when result
+        (let [result' (edn/read-string result)]
+          (when (seq result')
+            (when-let [conn (db-conn/get-db graph false)]
+              (let [tx-data (if (and (coll? result')
+                                     (coll? (first result'))
+                                     (not (map? (first result'))))
+                              (apply concat result')
+                              result')]
+                (try
+                  (d/transact! conn tx-data)
+                  (catch :default e
+                    (js/console.error "<q failed with:" e)
+                    nil)))))
+          result')))))
+
+(defn <pull
+  ([graph id]
+   (<pull graph '[*] id))
+  ([graph selector id]
+   (when-let [^Object sqlite @state/*db-worker]
+     (p/let [result (.pull sqlite graph (pr-str selector) (pr-str id))]
+       (when result
+         (let [result' (edn/read-string result)]
+           (when-let [conn (db-conn/get-db graph false)]
+             (d/transact! conn [result']))
+           result'))))))
+
+(comment
+  (defn <pull-many
+   [graph selector ids]
+   (assert (seq ids))
+   (when-let [^Object sqlite @state/*db-worker]
+     (p/let [result (.pull-many sqlite graph (pr-str selector) (pr-str ids))]
+       (when result
+         (edn/read-string result))))))

src/main/frontend/db/model.cljs

@@ -12,14 +12,11 @@
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
-            [frontend.util.drawer :as drawer]
             [logseq.db.frontend.rules :as rules]
             [logseq.db.frontend.content :as db-content]
             [logseq.graph-parser.text :as text]
             [logseq.graph-parser.util.db :as db-util]
             [logseq.common.util :as common-util]
-            [cljs-time.core :as t]
-            [cljs-time.format :as tf]
             [frontend.config :as config]
             [logseq.db :as ldb]))
 
@@ -117,18 +114,6 @@
   (when-let [db (conn/get-db repo)]
     (ldb/get-alias-source-page db alias)))
 
-(defn get-files
-  [repo]
-  (when-let [db (conn/get-db repo)]
-    (->> (d/q
-          '[:find ?path ?modified-at
-            :where
-            [?file :file/path ?path]
-            [(get-else $ ?file :file/last-modified-at 0) ?modified-at]]
-          db)
-         (seq)
-         (reverse))))
-
 (defn get-files-blocks
   [repo-url paths]
   (let [paths (set paths)
@@ -274,15 +259,7 @@ independent of format as format specific heading characters are stripped"
   [repo-url page]
   (when-let [page-id (:db/id (db-utils/entity repo-url [:block/name (util/safe-page-name-sanity-lc page)]))]
     (->>
-     (d/q '[:find ?e
-            :in $ ?page-name %
-            :where
-            [?page :block/name ?page-name]
-            (alias ?page ?e)]
-          (conn/get-db repo-url)
-          (util/safe-page-name-sanity-lc page)
-          (:alias rules/rules))
-     db-utils/seq-flatten
+     (ldb/get-page-alias (conn/get-db repo-url) page-id)
      (set)
      (set/union #{page-id}))))
 
@@ -336,25 +313,8 @@ independent of format as format specific heading characters are stripped"
     (->
      (react/q repo [:frontend.worker.react/block id]
               {:query-fn (fn [_]
-                           (let [e (db-utils/entity id)
-                                 children (map :db/id (sort-by-left (:block/_parent e) e))]
-                             [e {:name (:block/name e)
-                                 :original-name (:block/original-name e)
-                                 :link (:block/link e)
-                                 :namespace (:block/namespace e)
-                                 :types (:block/type e)
-                                 :schema (:block/schema e)
-                                 :content (:block/content e)
-                                 :marker (:block/marker e)
-                                 :priority (:block/priority e)
-                                 :properties (:block/properties e)
-                                 :properties-values (:block/properties-text-values e)
-                                 :alias (:block/alias e)
-                                 :tags (:block/tags e)
-                                 :children children
-                                 :collapsed? (:block/collapsed? e)
-                                 :collapsed-properties (:block/collapsed-properties e)
-                                 :refs-count (count (:block/_refs e))}]))}
+                           (let [e (db-utils/entity id)]
+                             [e (:block/tx-id e)]))}
               nil)
      react
      first)))
@@ -825,92 +785,26 @@ independent of format as format specific heading characters are stripped"
    (when repo
      (when (conn/get-db repo)
        (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)]))
-             pages (page-alias-set repo page)
-             aliases (set/difference pages #{page-id})]
+             pages (page-alias-set repo page)]
          (->>
           (react/q repo
-            [:frontend.worker.react/refs page-id]
-            {:use-cache? false
-             :query-fn (fn []
-                         (let [entities (mapcat (fn [id]
-                                                  (:block/_path-refs (db-utils/entity id))) pages)
-                               blocks (map (fn [e]
-                                             {:block/parent (:block/parent e)
-                                              :block/left (:block/left e)
-                                              :block/page (:block/page e)
-                                              :block/collapsed? (:block/collapsed? e)}) entities)]
-                           {:entities entities
-                            :blocks blocks}))}
-            nil)
+                   [:frontend.worker.react/refs page-id]
+                   {:query-fn (fn []
+                                (let [entities (mapcat (fn [id]
+                                                         (:block/_path-refs (db-utils/entity id))) pages)
+                                      blocks (map (fn [e]
+                                                    {:block/parent (:block/parent e)
+                                                     :block/left (:block/left e)
+                                                     :block/page (:block/page e)
+                                                     :block/collapsed? (:block/collapsed? e)}) entities)]
+                                  {:entities entities
+                                   :blocks blocks}))}
+                   nil)
           react
           :entities
-          (remove (fn [block] (= page-id (:db/id (:block/page block)))))))))))
-
-(defn get-date-scheduled-or-deadlines
-  [journal-title]
-  (when-let [date (date/journal-title->int journal-title)]
-    (let [future-days (state/get-scheduled-future-days)
-          date-format (tf/formatter "yyyyMMdd")
-          current-day (tf/parse date-format (str date))
-          future-day (some->> (t/plus current-day (t/days future-days))
-                              (tf/unparse date-format)
-                              (parse-long))]
-      (when future-day
-        (when-let [repo (state/get-current-repo)]
-          (->> (react/q repo [:custom :scheduled-deadline journal-title]
-                 {:use-cache? false}
-                 '[:find [(pull ?block ?block-attrs) ...]
-                   :in $ ?day ?future ?block-attrs
-                   :where
-                   (or
-                    [?block :block/scheduled ?d]
-                    [?block :block/deadline ?d])
-                   [(get-else $ ?block :block/repeated? false) ?repeated]
-                   [(get-else $ ?block :block/marker "NIL") ?marker]
-                   [(not= ?marker "DONE")]
-                   [(not= ?marker "CANCELED")]
-                   [(not= ?marker "CANCELLED")]
-                   [(<= ?d ?future)]
-                   (or-join [?repeated ?d ?day]
-                            [(true? ?repeated)]
-                            [(>= ?d ?day)])]
-                 date
-                 future-day
-                 block-attrs)
-               react
-               (sort-by-left-recursive)
-               db-utils/group-by-page))))))
-
-(defn- pattern [name]
-  (re-pattern (str "(?i)(^|[^\\[#0-9a-zA-Z]|((^|[^\\[])\\[))"
-                   (util/regex-escape name)
-                   "($|[^0-9a-zA-Z])")))
-
-(defn get-page-unlinked-references
-  [page]
-  (when-let [repo (state/get-current-repo)]
-    (let [page (util/safe-page-name-sanity-lc page)
-          page-id     (:db/id (db-utils/entity [:block/name page]))
-          alias-names (get-page-alias-names repo page)
-          patterns    (->> (conj alias-names page)
-                           (map pattern))
-          filter-fn   (fn [datom]
-                        (some (fn [p]
-                                (re-find p (->> (:v datom)
-                                                (drawer/remove-logbook))))
-                              patterns))]
-      (->> (react/q repo [:frontend.worker.react/page-unlinked-refs page-id]
-             {:query-fn (fn [db _result]
-                          (let [ids
-                                (->> (d/datoms db :aevt :block/content)
-                                     (filter filter-fn)
-                                     (map :e))
-                                result (db-utils/pull-many repo block-attrs ids)]
-                            (remove (fn [block] (= page-id (:db/id (:block/page block)))) result)))}
-             nil)
-           react
-           (sort-by-left-recursive)
-           db-utils/group-by-page))))
+          (remove (fn [block]
+                    (= page-id (:db/id (:block/page block)))))
+          (util/distinct-by :db/id)))))))
 
 (defn get-block-referenced-blocks
   ([block-uuid]
@@ -948,21 +842,6 @@ independent of format as format specific heading characters are stripped"
   [property-uuid]
   (ldb/get-classes-with-property (conn/get-db) property-uuid))
 
-(defn get-template-by-name
-  [name]
-  (when (string? name)
-    (->> (d/q
-          '[:find [(pull ?b [*]) ...]
-            :in $ ?name
-            :where
-            [?b :block/properties ?p]
-            [(get ?p :template) ?t]
-            [(= ?t ?name)]]
-          (conn/get-db)
-          name)
-         (sort-by :block/name)
-         (first))))
-
 (defn get-all-referenced-blocks-uuid
   "Get all uuids of blocks with any back link exists."
   []
@@ -976,7 +855,8 @@ independent of format as format specific heading characters are stripped"
 (defn delete-blocks
   [repo-url files _delete-page?]
   (when (seq files)
-    (let [blocks (get-files-blocks repo-url files)]
+    (let [blocks (->> (get-files-blocks repo-url files)
+                      (remove nil?))]
       (mapv (fn [eid] [:db.fn/retractEntity eid]) blocks))))
 
 (defn delete-files
@@ -999,6 +879,7 @@ independent of format as format specific heading characters are stripped"
                     :file/content content}]
        (db-utils/transact! repo [tx-data] (merge opts {:skip-refresh? true}))))))
 
+;; TODO: check whether this works when adding pdf back on Web
 (defn get-pre-block
   [repo page-id]
   (-> (d/q '[:find (pull ?b [*])

src/main/frontend/db/react.cljs

@@ -4,13 +4,15 @@
   It'll be great if we can find an automatically resolving and performant
   solution.
   "
-  (:require [datascript.core :as d]
-            [frontend.date :as date]
+  (:require [frontend.date :as date]
             [frontend.db.conn :as conn]
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
-            [clojure.core.async :as async]))
+            [clojure.core.async :as async]
+            [frontend.db.async.util :as db-async-util]
+            [promesa.core :as p]
+            [datascript.core :as d]))
 
 ;; Query atom of map of Key ([repo q inputs]) -> atom
 ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items?
@@ -43,15 +45,13 @@
   (reset! query-state {}))
 
 (defn add-q!
-  [k query time inputs result-atom transform-fn query-fn inputs-fn]
-  (let [time' (int (util/safe-parse-float time))] ;; for robustness. `time` should already be float
-    (swap! query-state assoc k {:query query
-                                :query-time time'
-                                :inputs inputs
-                                :result result-atom
-                                :transform-fn transform-fn
-                                :query-fn query-fn
-                                :inputs-fn inputs-fn}))
+  [k query inputs result-atom transform-fn query-fn inputs-fn]
+  (swap! query-state assoc k {:query query
+                              :inputs inputs
+                              :result result-atom
+                              :transform-fn transform-fn
+                              :query-fn query-fn
+                              :inputs-fn inputs-fn})
   result-atom)
 
 (defn remove-q!
@@ -85,16 +85,40 @@
   (when-let [result (get @query-state k)]
     (when (satisfies? IWithMeta @(:result result))
       (set! (.-state (:result result))
-           (with-meta @(:result result) {:query-time (:query-time result)})))
+            @(:result result)))
     (:result result)))
 
+(defn- <q-aux
+  [repo db query-fn inputs-fn k query inputs]
+  (let [kv? (and (vector? k) (= :kv (second k)))
+        journals? (and (vector? k) (= :frontend.worker.react/journals (last k)))
+        q (if (or journals? util/node-test?)
+            (fn [query inputs] (apply d/q query db inputs))
+            (fn [query inputs] (apply db-async-util/<q repo (cons query inputs))))]
+    (when (or query-fn query kv?)
+      (cond
+        query-fn
+        (query-fn db nil)
+
+        kv?
+        (db-utils/entity db (last k))
+
+        inputs-fn
+        (let [inputs (inputs-fn)]
+          (q query inputs))
+
+        (seq inputs)
+        (q query inputs)
+
+        :else
+        (q query nil)))))
+
 (defn q
-  [repo k {:keys [use-cache? transform-fn query-fn inputs-fn disable-reactive?]
+  [repo k {:keys [use-cache? transform-fn query-fn inputs-fn disable-reactive? return-promise?]
            :or {use-cache? true
                 transform-fn identity}} query & inputs]
   ;; {:pre [(s/valid? :frontend.worker.react/block k)]}
-  (let [kv? (and (vector? k) (= :kv (first k)))
-        origin-key k
+  (let [origin-key k
         k (vec (cons repo k))]
     (when-let [db (conn/get-db repo)]
       (let [result-atom (get-query-cached-result k)]
@@ -104,30 +128,26 @@
           (swap! queries conj origin-key))
         (if (and use-cache? result-atom)
           result-atom
-          (let [{:keys [result time]} (util/with-time
-                                        (-> (cond
-                                              query-fn
-                                              (query-fn db nil)
-
-                                              inputs-fn
-                                              (let [inputs (inputs-fn)]
-                                                (apply d/q query db inputs))
-
-                                              kv?
-                                              (db-utils/entity db (last k))
-
-                                              (seq inputs)
-                                              (apply d/q query db inputs)
-
-                                              :else
-                                              (d/q query db))
-                                            transform-fn))
-                result-atom (or result-atom (atom nil))]
-            ;; Don't notify watches now
-            (set! (.-state result-atom) result)
-            (if disable-reactive?
-              result-atom
-              (add-q! k query time inputs result-atom transform-fn query-fn inputs-fn))))))))
+          (let [result-atom (or result-atom (atom nil))
+                p-or-value (<q-aux repo db query-fn inputs-fn k query inputs)]
+            (when-not disable-reactive?
+              (add-q! k query inputs result-atom transform-fn query-fn inputs-fn))
+            (cond
+              return-promise?
+              p-or-value
+
+              (p/promise? p-or-value)
+              (do
+                (p/let [result p-or-value
+                        result' (transform-fn result)]
+                  (reset! result-atom result'))
+                result-atom)
+
+              :else
+              (let [result' (transform-fn p-or-value)]
+                ;; Don't notify watches now
+                (set! (.-state result-atom) result')
+                result-atom))))))))
 
 (defn get-current-page
   []
@@ -146,37 +166,14 @@
         (db-utils/entity [:block/name page-name])))))
 
 (defn- execute-query!
-  [graph db k {:keys [query _query-time inputs transform-fn query-fn inputs-fn result]}
-   {:keys [_skip-query-time-check?]}]
-  ;; FIXME:
-  (when true
-      ;; (or skip-query-time-check?
-      ;;       (<= (or query-time 0) 80))
-    (let [new-result (->
-                      (cond
-                        query-fn
-                        (let [result (query-fn db result)]
-                          (if (coll? result)
-                            (doall result)
-                            result))
-
-                        inputs-fn
-                        (let [inputs (inputs-fn)]
-                          (apply d/q query db inputs))
-
-                        (keyword? query)
-                        (db-utils/get-key-value graph query)
-
-                        (seq inputs)
-                        (apply d/q query db inputs)
-
-                        :else
-                        (d/q query db))
-                      transform-fn)]
-      (when-not (= new-result result)
-       (set-new-result! k new-result)))))
-
-(defn- refresh-affected-queries!
+  [graph db k {:keys [query inputs transform-fn query-fn inputs-fn result]
+               :or {transform-fn identity}}]
+  (p/let [p-or-value (<q-aux graph db query-fn inputs-fn k query inputs)
+          result' (transform-fn p-or-value)]
+    (when-not (= result' result)
+      (set-new-result! k result'))))
+
+(defn refresh-affected-queries!
   [repo-url affected-keys]
   (util/profile
    "refresh!"
@@ -198,7 +195,7 @@
                {:keys [custom-query?]} (state/edit-in-query-or-refs-component)]
            (when (or query query-fn)
              (try
-               (let [f #(execute-query! repo-url db (vec (cons repo-url k)) cache {:skip-query-time-check? custom-query?})]
+               (let [f #(execute-query! repo-url db (vec (cons repo-url k)) cache)]
                        ;; Detects whether user is editing in a custom query, if so, execute the query immediately
                  (if (and custom? (not custom-query?))
                    (async/put! (state/get-reactive-custom-queries-chan) [f query])

src/main/frontend/db/restore.cljs

@@ -6,27 +6,10 @@
             [frontend.persist-db :as persist-db]
             [promesa.core :as p]
             [cljs-time.core :as t]
-            [datascript.transit :as dt]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]))
-
-(comment
-  (defn- old-schema?
-    "Requires migration if the schema version is older than db-schema/version"
-    [db]
-    (let [v (db-migrate/get-schema-version db)
-        ;; backward compatibility
-          v (if (integer? v) v 0)]
-      (cond
-        (= db-schema/version v)
-        false
-
-        (< db-schema/version v)
-        (do
-          (js/console.error "DB schema version is newer than the app, please update the app. " ":db-version" v)
-          false)
-
-        :else
-        true))))
+            [logseq.db.sqlite.common-db :as sqlite-common-db]
+            [clojure.edn :as edn]
+            [frontend.db.async :as db-async]
+            [clojure.core.async :as async]))
 
 (defn restore-graph!
   "Restore db from SQLite"
@@ -35,23 +18,18 @@
   (p/let [start-time (t/now)
           data (persist-db/<fetch-init-data repo)
           _ (assert (some? data) "No data found when reloading db")
-          datoms (dt/read-transit-str data)
-          datoms-count (count datoms)
+          data' (edn/read-string data)
           db-schema (db-conn/get-schema repo)
-          conn (sqlite-common-db/restore-initial-data datoms db-schema)
+          conn (sqlite-common-db/restore-initial-data data' db-schema)
           db-name (db-conn/datascript-db repo)
           _ (swap! db-conn/conns assoc db-name conn)
           end-time (t/now)]
 
-    (println :restore-graph-from-sqlite!-prepare (t/in-millis (t/interval start-time end-time)) "ms"
-             " Datoms in total: " datoms-count)
-
-    ;; FIXME:
-    ;; (db-migrate/migrate attached-db)
+    (println ::restore-graph! "loads" (count data') "txs in" (t/in-millis (t/interval start-time end-time)) "ms")
 
-    (p/let [_ (p/delay 150)]          ; More time for UI refresh
-      (state/set-state! [repo :restore/unloaded-blocks] nil)
-      (state/set-state! [repo :restore/unloaded-pages] nil)
-      (state/set-state! :graph/loading? false)
-      (react/clear-query-state!)
-      (state/pub-event! [:ui/re-render-root]))))
+    (state/set-state! :graph/loading? false)
+    (react/clear-query-state!)
+    (state/pub-event! [:ui/re-render-root])
+    (async/go
+      (async/<! (async/timeout 100))
+      (db-async/<fetch-all-pages repo))))

src/main/frontend/db/utils.cljs

@@ -12,12 +12,6 @@
 (defn db->string [db]
   (dt/write-transit-str db))
 
-(defn db->json [db]
-  (js/JSON.stringify
-   (into-array
-    (for [d (d/datoms db :eavt)]
-      #js [(:e d) (name (:a d)) (:v d)]))))
-
 (defn db->edn-str [db]
   (pr-str db))
 

src/main/frontend/db_worker.cljs

@@ -6,7 +6,6 @@
             [datascript.core :as d]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
             [shadow.cljs.modern :refer [defclass]]
-            [datascript.transit :as dt]
             ["@logseq/sqlite-wasm" :default sqlite3InitModule]
             ["comlink" :as Comlink]
             [clojure.string :as string]
@@ -24,14 +23,15 @@
             [clojure.core.async :as async]
             [frontend.worker.async-util :include-macros true :refer [<?]]
             [frontend.worker.util :as worker-util]
-            [frontend.worker.handler.page.rename :as worker-page-rename]))
+            [frontend.worker.handler.page.rename :as worker-page-rename]
+            [frontend.worker.handler.page :as worker-page]
+            [logseq.outliner.op :as outliner-op]))
 
 (defonce *sqlite worker-state/*sqlite)
 (defonce *sqlite-conns worker-state/*sqlite-conns)
 (defonce *datascript-conns worker-state/*datascript-conns)
 (defonce *opfs-pools worker-state/*opfs-pools)
 (defonce *publishing? (atom false))
-(defonce *store-jobs (atom #{}))
 
 (defn- get-pool-name
   [graph-name]
@@ -109,11 +109,8 @@
                   (fn [[addr data]]
                     #js {:$addr addr
                          :$content (pr-str data)})
-                  addr+data-seq)
-            p (p/do! (upsert-addr-content! repo data delete-addrs))]
-        (swap! *store-jobs conj p)
-        (p/then p (fn [] (swap! *store-jobs disj p)))
-        p))
+                  addr+data-seq)]
+        (upsert-addr-content! repo data delete-addrs)))
 
     (-restore [_ addr]
       (restore-data-from-addr repo addr))))
@@ -264,12 +261,6 @@
    [_this repo & {:keys [close-other-db?]
                   :or {close-other-db? true}}]
    (p/do!
-    ;; Store the current db if store jobs not finished yet
-    (when (seq @*store-jobs)
-      (-> (p/all @*store-jobs)
-          (p/then (fn [_]
-                    (reset! *store-jobs #{})
-                    (println "DB store job finished")))))
     (when close-other-db?
       (close-other-dbs! repo))
     (create-or-open-db! repo)))
@@ -282,9 +273,62 @@
   (q [_this repo inputs-str]
      "Datascript q"
      (when-let [conn (worker-state/get-datascript-conn repo)]
-       (let [inputs (edn/read-string inputs-str)]
-         (let [result (apply d/q (first inputs) @conn (rest inputs))]
-           (bean/->js result)))))
+       (let [inputs (edn/read-string inputs-str)
+             result (apply d/q (first inputs) @conn (rest inputs))]
+         (pr-str result))))
+
+  (pull
+   [_this repo selector-str id-str]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [selector (edn/read-string selector-str)
+           id (edn/read-string id-str)
+           result (->> (d/pull @conn selector id)
+                       (sqlite-common-db/with-parent-and-left @conn))]
+       (pr-str result))))
+
+  (pull-many
+   [_this repo selector-str ids-str]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [selector (edn/read-string selector-str)
+           ids (edn/read-string ids-str)
+           result (d/pull-many @conn selector ids)]
+       (pr-str result))))
+
+  (get-right-sibling
+   [_this repo db-id]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [result (ldb/get-right-sibling @conn db-id)]
+       (pr-str result))))
+
+  (get-block-and-children
+   [_this repo name children?]
+   (assert (string? name))
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (pr-str (sqlite-common-db/get-block-and-children @conn name children?))))
+
+  (get-block-refs
+   [_this repo id]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (pr-str (ldb/get-block-refs @conn id))))
+
+  (get-block-refs-count
+   [_this repo id]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (ldb/get-block-refs-count @conn id)))
+
+  (get-block-parents
+   [_this repo id depth]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [block-id (:block/uuid (d/entity @conn id))
+           parents (->> (ldb/get-block-parents @conn block-id {:depth (or depth 3)})
+                        (map (fn [b] (d/pull @conn '[*] (:db/id b)))))]
+       (pr-str parents))))
+
+  (get-page-unlinked-refs
+   [_this repo page-id search-result-eids-str]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [search-result-eids (edn/read-string search-result-eids-str)]
+       (pr-str (ldb/get-page-unlinked-refs @conn page-id search-result-eids)))))
 
   (transact
    [_this repo tx-data tx-meta context]
@@ -333,8 +377,22 @@
   (getInitialData
    [_this repo]
    (when-let [conn (worker-state/get-datascript-conn repo)]
-     (->> (sqlite-common-db/get-initial-data @conn)
-          dt/write-transit-str)))
+     (pr-str (sqlite-common-db/get-initial-data @conn))))
+
+  (fetch-all-pages
+   [_this repo]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (async/go
+       (let [all-pages (sqlite-common-db/get-all-pages @conn)
+             partitioned-data (map-indexed (fn [idx p] [idx p]) (partition-all 2000 all-pages))]
+         (doseq [[idx tx-data] partitioned-data]
+           (worker-util/post-message :sync-db-changes (pr-str
+                                                       {:repo repo
+                                                        :tx-data tx-data
+                                                        :tx-meta {:initial-pages? true
+                                                                  :end? (= idx (dec (count partitioned-data)))}}))
+           (async/<! (async/timeout 100)))))
+     nil))
 
   (closeDB
    [_this repo]
@@ -401,6 +459,7 @@
    (when-let [conn (worker-state/get-datascript-conn repo)]
      (search/build-blocks-indice repo @conn)))
 
+  ;; page ops
   (page-search
    [this repo q limit]
    (when-let [conn (worker-state/get-datascript-conn repo)]
@@ -413,6 +472,29 @@
            result (worker-page-rename/rename! repo conn config old-name new-name)]
        (bean/->js {:result result}))))
 
+  (page-delete
+   [this repo page-name]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [result (worker-page/delete! repo conn page-name nil {})]
+       (bean/->js {:result result}))))
+
+  (apply-outliner-ops
+   [this repo ops-str opts-str]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [ops (edn/read-string ops-str)
+           opts (edn/read-string opts-str)
+           start-tx (:max-tx @conn)
+           result (outliner-op/apply-ops! repo conn ops (worker-state/get-date-formatter repo) opts)
+           end-tx (:max-tx @conn)]
+       (when (= start-tx end-tx)        ; nothing changes
+         ;; remove task from ldb/*request-id->response
+         (worker-util/post-message :sync-db-changes (pr-str
+                                                     {:request-id (:request-id opts)
+                                                      :repo repo
+                                                      :tx-data []
+                                                      :tx-meta nil})))
+       (pr-str result))))
+
   (file-writes-finished?
    [this repo]
    (let [conn (worker-state/get-datascript-conn repo)

src/main/frontend/extensions/tldraw.cljs

@@ -23,7 +23,8 @@
             [rum.core :as rum]
             [frontend.ui :as ui]
             [frontend.components.whiteboard :as whiteboard]
-            [cljs-bean.core :as bean]))
+            [cljs-bean.core :as bean]
+            [frontend.db.async :as db-async]))
 
 (def tldraw (r/adapt-class (gobj/get TldrawLogseq "App")))
 
@@ -205,21 +206,26 @@
             :model data})])
 
 (rum/defc tldraw-app-inner < rum/reactive
+  {:init (fn [state]
+           (let [page-name (first (:rum/args state))]
+             (db-async/<get-block (state/get-current-repo) page-name)
+             state))}
   [page-name block-id loaded-app set-loaded-app]
-  (let [populate-onboarding? (whiteboard-handler/should-populate-onboarding-whiteboard? page-name)
-        on-mount (fn [^js tln]
-                   (when tln
-                     (set! (.-appUndo tln) undo)
-                     (set! (.-appRedo tln) redo)
-                     (when-let [^js api (gobj/get tln "api")]
-                       (p/then (when populate-onboarding?
-                                 (whiteboard-handler/populate-onboarding-whiteboard api))
-                               #(do (whiteboard-handler/cleanup! (.-currentPage tln))
-                                    (state/focus-whiteboard-shape tln block-id)
-                                    (set-loaded-app tln))))))
-        data (whiteboard-handler/page-name->tldr! page-name)]
-    (when data
-      (tldraw-inner page-name data populate-onboarding? loaded-app on-mount))))
+  (when-not (state/sub-async-query-loading page-name)
+    (let [populate-onboarding? (whiteboard-handler/should-populate-onboarding-whiteboard? page-name)
+          on-mount (fn [^js tln]
+                     (when tln
+                       (set! (.-appUndo tln) undo)
+                       (set! (.-appRedo tln) redo)
+                       (when-let [^js api (gobj/get tln "api")]
+                         (p/then (when populate-onboarding?
+                                   (whiteboard-handler/populate-onboarding-whiteboard api))
+                                 #(do (whiteboard-handler/cleanup! (.-currentPage tln))
+                                      (state/focus-whiteboard-shape tln block-id)
+                                      (set-loaded-app tln))))))
+          data (whiteboard-handler/page-name->tldr! page-name)]
+      (when data
+        (tldraw-inner page-name data populate-onboarding? loaded-app on-mount)))))
 
 (rum/defc tldraw-app
   [page-name block-id]

src/main/frontend/extensions/zotero/handler.cljs

@@ -79,7 +79,7 @@
          (when-not (str/blank? page-name)
            (if (db/page-exists? (str/lower-case page-name))
              (if (setting/setting :overwrite-mode?)
-               (page-handler/delete!
+               (page-handler/<delete!
                 page-name
                 (fn [] (create-page page-name properties)))
                (editor-handler/api-insert-new-block!

src/main/frontend/external/roam_export.cljs

@@ -2,9 +2,9 @@
   (:require [clojure.set :as s]
             [clojure.string :as str]
             [clojure.walk :as walk]
-            [datascript.core :as d]
-            [frontend.db :as db]
-            [frontend.state :as state]))
+            [frontend.db.async :as db-async]
+            [frontend.state :as state]
+            [promesa.core :as p]))
 
 (def todo-marker-regex
   #"^(NOW|LATER|TODO|DOING|WAITING|WAIT|CANCELED|CANCELLED|STARTED|IN-PROGRESS)")
@@ -20,17 +20,18 @@
   (->> (repeatedly 9 nano-id-char)
        (str/join)))
 
-(defn uuid->uid-map []
-  (let [db (db/get-db (state/get-current-repo))]
-    (->>
-     (d/q '[:find (pull ?r [:block/uuid])
-            :in $
-            :where
-            [?b :block/refs ?r]] db)
-     (map (comp :block/uuid first))
-     (distinct)
-     (map (fn [uuid] [uuid (nano-id)]))
-     (into {}))))
+(defn <uuid->uid-map []
+  (let [repo (state/get-current-repo)]
+    (p/let [result (db-async/<q repo
+                                '[:find (pull ?r [:block/uuid])
+                                  :in $
+                                  :where
+                                  [?b :block/refs ?r]])]
+      (->> result
+       (map (comp :block/uuid first))
+       (distinct)
+       (map (fn [uuid] [uuid (nano-id)]))
+       (into {})))))
 
 (defn update-content [content uuid->uid-map]
   (when content                         ; page block doesn't have content
@@ -65,7 +66,7 @@
 
 (defn traverse
   [keyseq vec-tree]
-  (let [uuid->uid-map (uuid->uid-map)]
+  (p/let [uuid->uid-map (<uuid->uid-map)]
     (walk/postwalk
      (fn [x]
        (cond

src/main/frontend/fs/watcher_handler.cljs

@@ -3,12 +3,10 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [frontend.config :as config]
-            [frontend.date :as date]
             [frontend.db :as db]
             [frontend.db.model :as model]
             [frontend.fs :as fs]
             [logseq.common.path :as path]
-            [frontend.handler.editor :as editor-handler]
             [frontend.handler.file :as file-handler]
             [frontend.handler.file-based.property :as file-property-handler]
             [frontend.handler.global-config :as global-config-handler]
@@ -16,12 +14,12 @@
             [frontend.handler.page :as page-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.state :as state]
-            [frontend.util :as util]
             [frontend.util.fs :as fs-util]
             [lambdaisland.glogi :as log]
             [logseq.common.config :as common-config]
             [logseq.common.util.block-ref :as block-ref]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.db.async :as db-async]))
 
 ;; all IPC paths must be normalized! (via common-util/path-normalize)
 
@@ -69,9 +67,9 @@
           {:keys [mtime]} stat
           ext (keyword (path/file-ext path))]
       (when (contains? #{:org :md :markdown :css :js :edn :excalidraw :tldr} ext)
-        (let [db-content (db/get-file repo path)
-              exists-in-db? (not (nil? db-content))
-              db-content (or db-content "")]
+        (p/let [db-content (db-async/<get-file repo path)
+                exists-in-db? (not (nil? db-content))
+                db-content (or db-content "")]
           (when (or content (contains? #{"unlink" "unlinkDir" "addDir"} type))
             (cond
               (and (= "unlinkDir" type) dir)
@@ -107,7 +105,7 @@
                 (when dir-exists?
                   (when-let [page-name (db/get-file-page path)]
                     (println "Delete page: " page-name ", file path: " path ".")
-                    (page-handler/delete! page-name #()))))
+                    (page-handler/<delete! page-name #()))))
 
           ;; global config handling
               (and (= "change" type)
@@ -136,68 +134,15 @@
       ;; return nil, otherwise the entire db will be transferred by ipc
       nil)))
 
-(defn preload-graph-homepage-files!
-  "Preload the homepage file for the current graph. Return loaded file paths.
-
-   Prerequisites:
-   - current graph is set
-   - config is loaded"
-  []
-  (when-let [repo (state/get-current-repo)]
-    (when (and (not (state/loading-files? repo))
-               (config/local-file-based-graph? repo))
-      (let [repo-dir (config/get-repo-dir repo)
-            page-name (if (state/enable-journals? repo)
-                        (date/today)
-                        (or (:page (state/get-default-home)) "Contents"))
-            page-name (util/page-name-sanity-lc page-name)
-            file-rpath (or (:file/path (db/get-page-file page-name))
-                           (let [format (state/get-preferred-format repo)
-                                 ext (config/get-file-extension format)
-                                 file-name (if (state/enable-journals? repo)
-                                             (date/journal-title->default (date/today))
-                                             (or (:page (state/get-default-home)) "contents"))
-                                 parent-dir (if (state/enable-journals? repo)
-                                              (config/get-journals-directory)
-                                              (config/get-pages-directory))]
-                             (str parent-dir "/" file-name "." ext)))]
-        (prn ::preload-homepage file-rpath)
-        (p/let [file-exists? (fs/file-exists? repo-dir file-rpath)
-                _ (when file-exists?
-                    ;; BUG: avoid active-editing block content overwrites incoming fs changes
-                    (editor-handler/escape-editing false))
-                file-content (when file-exists?
-                               (fs/read-file repo-dir file-rpath))
-                file-mtime (when file-exists?
-                             (:mtime (fs/stat repo-dir file-rpath)))
-                db-empty? (db/page-empty? repo page-name)
-                db-content (if-not db-empty?
-                             (db/get-file repo file-rpath)
-                             "")]
-          (p/let [_ (cond
-                      (and file-exists?
-                           db-empty?)
-                      (handle-add-and-change! repo file-rpath file-content db-content file-mtime false)
-
-                      (and file-exists?
-                           (not db-empty?)
-                           (not= file-content db-content))
-                      (handle-add-and-change! repo file-rpath file-content db-content file-mtime true))]
-
-            (ui-handler/re-render-root!)
-
-            [file-rpath]))))))
-
 (defn load-graph-files!
   "This fn replaces the former initial fs watcher"
-  [graph exclude-files]
+  [graph]
   (when graph
-    (let [repo-dir (config/get-repo-dir graph)
-          db-files (->> (db/get-files graph)
-                        (map first))
-          exclude-files (set (or exclude-files []))]
+    (let [repo-dir (config/get-repo-dir graph)]
       ;; read all files in the repo dir, notify if readdir error
-      (p/let [[files deleted-files]
+      (p/let [db-files' (db-async/<get-files graph)
+              db-files (map first db-files')
+              [files deleted-files]
               (-> (fs/readdir repo-dir :path-only? true)
                   (p/chain (fn [files]
                              (->> files
@@ -209,8 +154,7 @@
                                                     (string/lower-case f)]))))
                            (fn [files]
                              (let [deleted-files (set/difference (set db-files) (set files))]
-                               [(->> files
-                                     (remove #(contains? exclude-files %)))
+                               [files
                                 deleted-files])))
                   (p/catch (fn [error]
                              (when-not (config/demo-graph? graph)

src/main/frontend/handler.cljs

@@ -11,7 +11,6 @@
             [frontend.components.whiteboard :as whiteboard]
             [frontend.config :as config]
             [frontend.context.i18n :as i18n]
-            [frontend.db :as db]
             [frontend.db.restore :as db-restore]
             [frontend.db.conn :as conn]
             [frontend.db.react :as react]
@@ -42,7 +41,8 @@
             [frontend.mobile.core :as mobile]
             [cljs-bean.core :as bean]
             [frontend.handler.test :as test]
-            [frontend.persist-db.browser :as db-browser]))
+            [frontend.persist-db.browser :as db-browser]
+            [frontend.db.async :as db-async]))
 
 (defn- set-global-error-notification!
   []
@@ -88,17 +88,18 @@
                 ;; install after config is restored
                 (shortcut/refresh!)
 
-                (cond
-                  (and (not (seq (db/get-files config/demo-repo)))
-                       ;; Not native local directory
-                       (not (some config/local-file-based-graph? (map :url repos)))
-                       (not (mobile-util/native-platform?))
-                       (not (config/db-based-graph? repo)))
-                  ;; will execute `(state/set-db-restoring! false)` inside
-                  (file-repo-handler/setup-demo-repo-if-not-exists!)
-
-                  :else
-                  (state/set-db-restoring! false)))))))
+                (p/let [files (db-async/<get-files config/demo-repo)]
+                  (cond
+                    (and (not (seq files))
+                         ;; Not native local directory
+                         (not (some config/local-file-based-graph? (map :url repos)))
+                         (not (mobile-util/native-platform?))
+                         (not (config/db-based-graph? repo)))
+                    ;; will execute `(state/set-db-restoring! false)` inside
+                    (file-repo-handler/setup-demo-repo-if-not-exists!)
+
+                    :else
+                    (state/set-db-restoring! false))))))))
         (p/then
          (fn []
            (js/console.log "db restored, setting up repo hooks")

src/main/frontend/handler/block.cljs

@@ -8,6 +8,7 @@
    [frontend.mobile.haptics :as haptics]
    [logseq.outliner.core :as outliner-core]
    [frontend.modules.outliner.ui :as ui-outliner-tx]
+   [frontend.modules.outliner.op :as outliner-op]
    [frontend.state :as state]
    [frontend.util :as util]
    [frontend.util.drawer :as drawer]
@@ -326,17 +327,18 @@
                                      last)]
        (get-original-block-by-dom last-block-node)))))
 
-(defn indent-outdent-block!
-  [block direction]
-  (ui-outliner-tx/transact!
-   {:outliner-op :move-blocks
-    :real-outliner-op :indent-outdent}
-   (outliner-core/indent-outdent-blocks! (state/get-current-repo)
-                                         (db/get-db false)
-                                         (get-top-level-blocks [block])
-                                         (= direction :right)
-                                         {:get-first-block-original get-first-block-original
-                                          :logical-outdenting? (state/logical-outdenting?)})))
+(defn indent-outdent-blocks!
+  [blocks indent? save-current-block]
+  (when (seq blocks)
+    (let [blocks (get-top-level-blocks blocks)]
+      (ui-outliner-tx/transact!
+       {:outliner-op :move-blocks
+        :real-outliner-op :indent-outdent}
+       (when save-current-block (save-current-block))
+       (outliner-op/indent-outdent-blocks! (get-top-level-blocks blocks)
+                                           indent?
+                                           {:parent-original (get-first-block-original)
+                                            :logical-outdenting? (state/logical-outdenting?)})))))
 
 (def *swipe (atom nil))
 
@@ -461,13 +463,13 @@
             (and left-menu (>= (.-clientWidth left-menu) 40))
             (when (indentable? block)
               (haptics/with-haptics-impact
-                (indent-outdent-block! block :right)
+                (indent-outdent-blocks! [block] true nil)
                 :light))
 
             (and right-menu (<= 40 (.-clientWidth right-menu) 79))
             (when (outdentable? block)
               (haptics/with-haptics-impact
-                (indent-outdent-block! block :left)
+                (indent-outdent-blocks! [block] false nil)
                 :light))
 
             (and right-menu (>= (.-clientWidth right-menu) 80))

src/main/frontend/handler/common/page.cljs

@@ -21,7 +21,7 @@
             [frontend.db.conn :as conn]
             [datascript.core :as d]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
-            [logseq.outliner.core :as outliner-core]))
+            [frontend.modules.outliner.op :as outliner-op]))
 
 (defn build-hidden-page-tx-data
   [page-name]
@@ -126,41 +126,41 @@
 (defn <favorite-page!-v2
   [page-block-uuid]
   {:pre [(uuid? page-block-uuid)]}
-  (let [repo (state/get-current-repo)
-        favorites-page (d/entity (conn/get-db) [:block/name favorites-page-name])
+  (let [favorites-page (d/entity (conn/get-db) [:block/name favorites-page-name])
         favorites-page-tx-data (build-hidden-page-tx-data "favorites")]
     (when (d/entity (conn/get-db) [:block/uuid page-block-uuid])
       (p/do!
        (when-not favorites-page (ldb/transact! nil [favorites-page-tx-data]))
        (ui-outliner-tx/transact!
         {:outliner-op :insert-blocks}
-        (outliner-core/insert-blocks! repo (conn/get-db false) [{:block/link [:block/uuid page-block-uuid]
-                                                                 :block/content ""
-                                                                 :block/format :markdown}]
-                                      (d/entity (conn/get-db) [:block/name favorites-page-name])
-                                      {}))))))
+        (outliner-op/insert-blocks! [{:block/link [:block/uuid page-block-uuid]
+                                      :block/content ""
+                                      :block/format :markdown}]
+                                    (d/entity (conn/get-db) [:block/name favorites-page-name])
+                                    {}))))))
 
 (defn <unfavorite-page!-v2
   [page-block-uuid]
   {:pre [(uuid? page-block-uuid)]}
-  (let [repo (state/get-current-repo)]
-    (when-let [block (find-block-in-favorites-page page-block-uuid)]
-      (ui-outliner-tx/transact!
-       {:outliner-op :delete-blocks}
-       (outliner-core/delete-blocks! repo (conn/get-db false) (state/get-date-formatter) [block] {})))))
+  (when-let [block (find-block-in-favorites-page page-block-uuid)]
+    (ui-outliner-tx/transact!
+     {:outliner-op :delete-blocks}
+     (outliner-op/delete-blocks! [block] {}))))
 
 
 ;; favorites fns end ================
 
 
-(defn delete!
+(defn <delete!
   "Deletes a page and then either calls the ok-handler or the error-handler if unable to delete"
-  [page-name ok-handler & {:keys [_persist-op? _error-handler]
-                           :as opts}]
+  [page-name ok-handler & {:keys [error-handler]}]
   (when page-name
-    (when-let [repo (state/get-current-repo)]
-      (let [conn (db/get-db repo false)]
-        (worker-page/delete! repo conn page-name ok-handler opts)))))
+    (when-let [^Object worker @state/*db-worker]
+      (-> (p/let [repo (state/get-current-repo)
+                  _ (.page-delete worker repo page-name)]
+            (when ok-handler (ok-handler)))
+          (p/catch (fn [error]
+                     (when error-handler (error-handler error))))))))
 
 ;; other fns
 ;; =========

src/main/frontend/handler/db_based/editor.cljs

@@ -14,8 +14,8 @@
             [frontend.handler.property :as property-handler]
             [frontend.handler.property.util :as pu]
             [frontend.handler.repo-config :as repo-config-handler]
-            [logseq.outliner.core :as outliner-core]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
+            [frontend.modules.outliner.op :as outliner-op]
             [frontend.schema.handler.repo-config :as repo-config-schema]
             [promesa.core :as p]
             [logseq.db.frontend.content :as db-content]))
@@ -161,6 +161,6 @@
   [repo block-ids heading]
   (ui-outliner-tx/transact!
    {:outliner-op :save-block}
-   (doseq [block-tx (keep #(set-heading-aux! % heading) block-ids)]
-     (outliner-core/save-block! repo (db/get-db false) (state/get-date-formatter) block-tx))
+   (doseq [block (keep #(set-heading-aux! % heading) block-ids)]
+     (outliner-op/save-block! block))
    (property-handler/batch-set-block-property! repo block-ids :heading heading)))

src/main/frontend/handler/dnd.cljs

@@ -2,9 +2,10 @@
   "Provides fns for drag and drop"
   (:require [frontend.handler.editor :as editor-handler]
             [frontend.handler.property :as property-handler]
-            [logseq.outliner.core :as outliner-core]
             [logseq.outliner.tree :as otree]
+            [logseq.outliner.core :as outliner-core]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
+            [frontend.modules.outliner.op :as outliner-op]
             [logseq.common.util.block-ref :as block-ref]
             [frontend.state :as state]
             [frontend.db :as db]
@@ -12,8 +13,7 @@
 
 (defn move-blocks
   [^js event blocks target-block original-block move-to]
-  (let [repo (state/get-current-repo)
-        blocks' (map #(db/pull (:db/id %)) blocks)
+  (let [blocks' (map #(db/pull (:db/id %)) blocks)
         first-block (first blocks')
         top? (= move-to :top)
         nested? (= move-to :nested)
@@ -53,10 +53,10 @@
                     (otree/-get-left-id target-node conn))]
              (if first-child?
                (when-let [parent (otree/-get-parent target-node conn)]
-                 (outliner-core/move-blocks! repo conn blocks' (:data parent) false))
+                 (outliner-op/move-blocks! blocks' (:data parent) false))
                (when-let [before-node (otree/-get-left target-node conn)]
-                 (outliner-core/move-blocks! repo conn blocks' (:data before-node) true))))
-           (outliner-core/move-blocks! repo conn blocks' target-block (not nested?)))))
+                 (outliner-op/move-blocks! blocks' (:data before-node) true))))
+           (outliner-op/move-blocks! blocks' target-block (not nested?)))))
 
       :else
       nil)))

src/main/frontend/handler/editor.cljs

@@ -9,6 +9,7 @@
             [frontend.db :as db]
             [frontend.db.model :as db-model]
             [frontend.db.utils :as db-utils]
+            [frontend.db.async :as db-async]
             [frontend.db.query-dsl :as query-dsl]
             [frontend.diff :as diff]
             [frontend.format.block :as block]
@@ -32,6 +33,7 @@
             [frontend.handler.file-based.editor :as file-editor-handler]
             [frontend.mobile.util :as mobile-util]
             [logseq.outliner.core :as outliner-core]
+            [frontend.modules.outliner.op :as outliner-op]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.modules.outliner.tree :as tree]
             [logseq.outliner.tree :as otree]
@@ -65,7 +67,8 @@
             [promesa.core :as p]
             [rum.core :as rum]
             [frontend.handler.db-based.property :as db-property-handler]
-            [frontend.fs.capacitor-fs :as capacitor-fs]))
+            [frontend.fs.capacitor-fs :as capacitor-fs]
+            [clojure.edn :as edn]))
 
 ;; FIXME: should support multiple images concurrently uploading
 
@@ -77,9 +80,7 @@
 
 (defn- outliner-save-block!
   [block]
-  (let [repo (state/get-current-repo)
-        conn (db/get-db false)]
-    (outliner-core/save-block! repo conn (state/get-date-formatter) block)))
+  (outliner-op/save-block! block))
 
 (defn get-block-own-order-list-type
   [block]
@@ -344,15 +345,13 @@
 
                    :else
                    (not has-children?))]
-    (p/do!
-     (ui-outliner-tx/transact!
-      {:outliner-op :insert-blocks}
-       (save-current-block! {:current-block current-block})
-       (outliner-core/insert-blocks! (state/get-current-repo) (db/get-db false)
-                                    [new-block] current-block {:sibling? sibling?
-                                                               :keep-uuid? keep-uuid?
-                                                               :ordered-list? ordered-list?
-                                                               :replace-empty-target? replace-empty-target?})))))
+    (ui-outliner-tx/transact!
+     {:outliner-op :insert-blocks}
+     (save-current-block! {:current-block current-block})
+     (outliner-op/insert-blocks! [new-block] current-block {:sibling? sibling?
+                                                            :keep-uuid? keep-uuid?
+                                                            :ordered-list? ordered-list?
+                                                            :replace-empty-target? replace-empty-target?}))))
 
 
 (defn- block-self-alone-when-insert?
@@ -553,10 +552,10 @@
                                                            :ordered-list? ordered-list?
                                                            :replace-empty-target? replace-empty-target?})
              (when edit-block?
-              (if (and replace-empty-target?
-                       (string/blank? (:block/content last-block)))
-                (edit-block! last-block :max nil)
-                (edit-block! new-block :max nil)))
+               (if (and replace-empty-target?
+                        (string/blank? (:block/content last-block)))
+                 (edit-block! last-block :max nil)
+                 (edit-block! new-block :max nil)))
              new-block)))))))
 
 (defn insert-first-page-block-if-not-exists!
@@ -688,12 +687,10 @@
         (state/set-state! :ui/deleting-block uuid)
         (ui-outliner-tx/transact!
          {:outliner-op :delete-blocks}
-         (outliner-core/delete-blocks! repo (db/get-db false)
-                                       (state/get-date-formatter)
-                                       blocks
-                                       (merge
-                                        delete-opts
-                                        {:children? children?})))))))
+         (outliner-op/delete-blocks! blocks
+                                     (merge
+                                      delete-opts
+                                      {:children? children?})))))))
 
 (defn- move-to-prev-block
   [repo sibling-block format _id value]
@@ -776,7 +773,8 @@
                         {:keys [prev-block new-content]} (move-to-prev-block repo sibling-block format id value)
                         concat-prev-block? (boolean (and prev-block new-content))
                         transact-opts {:outliner-op :delete-blocks}
-                        db-based? (config/db-based-graph? repo)]
+                        db-based? (config/db-based-graph? repo)
+                        db (db/get-db repo)]
                     (ui-outliner-tx/transact!
                      transact-opts
                      (cond
@@ -789,7 +787,7 @@
                        (let [new-properties (merge (:block/properties (db/entity (:db/id prev-block)))
                                                    (:block/properties (db/entity (:db/id block))))]
                          (if (seq (:block/_refs (db/entity (:db/id block))))
-                           (let [block-right (outliner-core/get-right-sibling (db/get-db) (:db/id block))]
+                           (do
                              (delete-block-fn prev-block)
                              (save-block! repo block new-content {})
                              (outliner-save-block! {:db/id (:db/id block)
@@ -798,10 +796,11 @@
                                                                     (:db/id (:block/parent prev-block)))})
 
                              ;; block->right needs to point its `left` to block->left
-                             (when (and block-right (not= (:db/id (:block/parent prev-block))
-                                                          (:db/id (:block/parent block))))
-                               (outliner-save-block! {:db/id (:db/id block-right)
-                                                      :block/left (:db/id (:block/left block))}))
+                             (let [block-right (outliner-core/get-right-sibling db (:db/id block))]
+                               (when (and block-right (not= (:db/id (:block/parent prev-block))
+                                                           (:db/id (:block/parent block))))
+                                (outliner-save-block! {:db/id (:db/id block-right)
+                                                       :block/left (:db/id (:block/left block))})))
 
                              ;; update prev-block's children to point to the refed block
                              (when (or (:block/collapsed? prev-block)
@@ -815,7 +814,7 @@
 
                              ;; parent will be removed
                              (when (= (:db/id prev-block) (:db/id (:block/parent block)))
-                               (when-let [parent-right (outliner-core/get-right-sibling (db/get-db) (:db/id prev-block))]
+                               (when-let [parent-right (when prev-block (outliner-core/get-right-sibling db (:db/id prev-block)))]
                                  (outliner-save-block! {:db/id (:db/id parent-right)
                                                         :block/left (:db/id block)})))
 
@@ -844,9 +843,7 @@
       (p/do!
        (ui-outliner-tx/transact!
         {:outliner-op :delete-blocks}
-        (outliner-core/delete-blocks! repo (db/get-db false)
-                                      (state/get-date-formatter)
-                                      blocks' {}))
+         (outliner-op/delete-blocks! blocks' nil))
        (when sibling-block
          (move-to-prev-block repo sibling-block
                              (:block/format block)
@@ -1744,7 +1741,7 @@
                        (let [blocks' (block-handler/get-top-level-blocks blocks)
                              result (ui-outliner-tx/transact!
                                      {:outliner-op :move-blocks}
-                                     (outliner-core/move-blocks-up-down! (state/get-current-repo) (db/get-db false) blocks' up?))]
+                                      (outliner-op/move-blocks-up-down! blocks' up?))]
                          (when-let [block-node (util/get-first-block-by-id (:block/uuid (first blocks)))]
                            (.scrollIntoView block-node #js {:behavior "smooth" :block "nearest"}))
                          result))]
@@ -1781,16 +1778,7 @@
   "`direction` = :left | :right."
   [direction]
   (let [blocks (get-selected-ordered-blocks)]
-    (when (seq blocks)
-      (ui-outliner-tx/transact!
-       {:outliner-op :move-blocks
-        :real-outliner-op :indent-outdent}
-       (outliner-core/indent-outdent-blocks! (state/get-current-repo)
-                                             (db/get-db false)
-                                             (block-handler/get-top-level-blocks blocks)
-                                             (= direction :right)
-                                             {:get-first-block-original block-handler/get-first-block-original
-                                              :logical-outdenting? (state/logical-outdenting?)})))))
+    (block-handler/indent-outdent-blocks! blocks (= direction :right) nil)))
 
 (defn- get-link [format link label]
   (let [link (or link "")
@@ -2052,7 +2040,7 @@
         page (if (:block/name block) block
                  (when target-block (:block/page (db/entity (:db/id target-block)))))
         empty-target? (if (true? skip-empty-target?) false
-                        (string/blank? (:block/content target-block)))
+                          (string/blank? (:block/content target-block)))
         paste-nested-blocks? (nested-blocks blocks)
         target-block-has-children? (db/has-children? (:block/uuid target-block))
         replace-empty-target? (and empty-target?
@@ -2080,23 +2068,21 @@
        {:outliner-op :save-block}
        (outliner-save-block! editing-block)))
 
-    (p/let [*insert-result (atom nil)
-            _ (ui-outliner-tx/transact!
-               {:outliner-op :insert-blocks
-                :additional-tx revert-cut-txs}
-               (when target-block'
-                 (let [format (or (:block/format target-block') (state/get-preferred-format))
-                       repo (state/get-current-repo)
-                       blocks' (map (fn [block]
-                                      (paste-block-cleanup repo block page exclude-properties format content-update-fn keep-uuid?))
-                                    blocks)
-                       result (outliner-core/insert-blocks! repo (db/get-db false) blocks' target-block' {:sibling? sibling?
-                                                                                                          :outliner-op :paste
-                                                                                                          :replace-empty-target? replace-empty-target?
-                                                                                                          :keep-uuid? keep-uuid?})]
-                   (reset! *insert-result result))))]
+    (p/let [result (ui-outliner-tx/transact!
+                    {:outliner-op :insert-blocks
+                     :additional-tx revert-cut-txs}
+                    (when target-block'
+                      (let [format (or (:block/format target-block') (state/get-preferred-format))
+                            repo (state/get-current-repo)
+                            blocks' (map (fn [block]
+                                           (paste-block-cleanup repo block page exclude-properties format content-update-fn keep-uuid?))
+                                         blocks)]
+                        (outliner-op/insert-blocks! blocks' target-block' {:sibling? sibling?
+                                                                           :outliner-op :paste
+                                                                           :replace-empty-target? replace-empty-target?
+                                                                           :keep-uuid? keep-uuid?}))))]
       (state/set-block-op-type! nil)
-      (when-let [result @*insert-result] (edit-last-block-after-inserted! result)))))
+      (when result (edit-last-block-after-inserted! (edn/read-string result))))))
 
 (defn- block-tree->blocks
   "keep-uuid? - maintain the existing :uuid in tree vec"
@@ -2150,70 +2136,66 @@
    (let [repo (state/get-current-repo)
          db? (config/db-based-graph? repo)]
      (when-not db?
-       (when-let [db-id (if (integer? db-id)
-                          db-id
-                          (:db/id (db-model/get-template-by-name (name db-id))))]
-         (let [journal? (:block/journal? target)
-               target (or target (state/get-edit-block))
-               block (db/entity db-id)
-               format (:block/format block)
-               block-uuid (:block/uuid block)
-               template-including-parent? (not (false? (:template-including-parent (:block/properties block))))
-               blocks (db/get-block-and-children repo block-uuid)
-               root-block (db/pull db-id)
-               blocks-exclude-root (remove (fn [b] (= (:db/id b) db-id)) blocks)
-               sorted-blocks (tree/sort-blocks blocks-exclude-root root-block)
-               sorted-blocks (cons
-                              (-> (first sorted-blocks)
-                                  (update :block/properties-text-values dissoc :template)
-                                  (update :block/properties-order (fn [keys]
-                                                                    (vec (remove #{:template} keys)))))
-                              (rest sorted-blocks))
-               blocks (if template-including-parent?
-                        sorted-blocks
-                        (drop 1 sorted-blocks))]
-           (when element-id
-             (insert-command! element-id "" format {:end-pattern commands/command-trigger}))
-           (let [exclude-properties [:id :template :template-including-parent]
-                 content-update-fn (fn [content]
-                                     (->> content
-                                          (property-file/remove-property-when-file-based repo format "template")
-                                          (property-file/remove-property-when-file-based repo format "template-including-parent")
-                                          template/resolve-dynamic-template!))
-                 page (if (:block/name block) block
-                          (when target (:block/page (db/entity (:db/id target)))))
-                 blocks' (map (fn [block]
-                                (paste-block-cleanup repo block page exclude-properties format content-update-fn false))
-                              blocks)
-                 sibling? (:sibling? opts)
-                 sibling?' (cond
-                             (some? sibling?)
-                             sibling?
-
-                             (db/has-children? (:block/uuid target))
-                             false
-
-                             :else
-                             true)]
-             (try
-               (let [*result (atom nil)]
-                 (p/do!
-                  (ui-outliner-tx/transact!
-                   {:outliner-op :insert-blocks
-                    :created-from-journal-template? journal?}
-                   (when-not (string/blank? (state/get-edit-content))
-                     (save-current-block!))
-                   (let [result (outliner-core/insert-blocks! repo (db/get-db false) blocks'
-                                                              target
-                                                              (assoc opts :sibling? sibling?'))]
-                     (reset! *result result)))
-                  (some-> @*result edit-last-block-after-inserted!)))
-
-               (catch :default ^js/Error e
-                 (notification/show!
-                  [:p.content
-                   (util/format "Template insert error: %s" (.-message e))]
-                  :error))))))))))
+       (let [block (if (integer? db-id)
+                     (db-async/<pull repo db-id)
+                     (db-async/<get-template-by-name (name db-id)))]
+         (when-let [db-id (:db/id block)]
+           (let [journal? (:block/journal? target)
+                 target (or target (state/get-edit-block))
+                 format (:block/format block)
+                 block-uuid (:block/uuid block)
+                 template-including-parent? (not (false? (:template-including-parent (:block/properties block))))
+                 blocks (db/get-block-and-children repo block-uuid)
+                 root-block (db/pull db-id)
+                 blocks-exclude-root (remove (fn [b] (= (:db/id b) db-id)) blocks)
+                 sorted-blocks (tree/sort-blocks blocks-exclude-root root-block)
+                 sorted-blocks (cons
+                                (-> (first sorted-blocks)
+                                    (update :block/properties-text-values dissoc :template)
+                                    (update :block/properties-order (fn [keys]
+                                                                      (vec (remove #{:template} keys)))))
+                                (rest sorted-blocks))
+                 blocks (if template-including-parent?
+                          sorted-blocks
+                          (drop 1 sorted-blocks))]
+             (when element-id
+               (insert-command! element-id "" format {:end-pattern commands/command-trigger}))
+             (let [exclude-properties [:id :template :template-including-parent]
+                   content-update-fn (fn [content]
+                                       (->> content
+                                            (property-file/remove-property-when-file-based repo format "template")
+                                            (property-file/remove-property-when-file-based repo format "template-including-parent")
+                                            template/resolve-dynamic-template!))
+                   page (if (:block/name block) block
+                            (when target (:block/page (db/entity (:db/id target)))))
+                   blocks' (map (fn [block]
+                                  (paste-block-cleanup repo block page exclude-properties format content-update-fn false))
+                                blocks)
+                   sibling? (:sibling? opts)
+                   sibling?' (cond
+                               (some? sibling?)
+                               sibling?
+
+                               (db/has-children? (:block/uuid target))
+                               false
+
+                               :else
+                               true)]
+               (try
+                 (p/let [result (ui-outliner-tx/transact!
+                                 {:outliner-op :insert-blocks
+                                  :created-from-journal-template? journal?}
+                                 (when-not (string/blank? (state/get-edit-content))
+                                   (save-current-block!))
+                                 (outliner-op/insert-blocks! blocks' target
+                                                             (assoc opts :sibling? sibling?')))]
+                   (when result (edit-last-block-after-inserted! (edn/read-string result))))
+
+                 (catch :default ^js/Error e
+                   (notification/show!
+                    [:p.content
+                     (util/format "Template insert error: %s" (.-message e))]
+                    :error)))))))))))
 
 (defn template-on-chosen-handler
   [element-id]
@@ -2272,9 +2254,8 @@
        :real-outliner-op :indent-outdent}
       (save-current-block!)
       (when target
-        (outliner-core/move-blocks! (state/get-current-repo) (db/get-db false)
-                                    (block-handler/get-top-level-blocks [block])
-                                    target true)))
+        (outliner-op/move-blocks! (block-handler/get-top-level-blocks [block])
+                                  target true)))
      (when original-block
        (util/schedule #(edit-block! block pos nil))))))
 
@@ -2694,15 +2675,15 @@
   (state/set-edit-content! (state/get-edit-input-id) (.-value input)))
 
 (defn- delete-concat [current-block]
-  (let [repo (state/get-current-repo)
-        ^js input (state/get-input)
-        current-pos (cursor/pos input)
-        value (gobj/get input "value")
-        collapsed? (util/collapsed? current-block)
-        next-block (when-let [e (db-model/get-next (db/get-db repo) (:db/id current-block))]
-                     (db/pull (:db/id e)))
-        next-block-right (when next-block (outliner-core/get-right-sibling (db/get-db) (:db/id next-block)))
-        db-based? (config/db-based-graph? repo)]
+  (p/let [repo (state/get-current-repo)
+          ^js input (state/get-input)
+          current-pos (cursor/pos input)
+          value (gobj/get input "value")
+          collapsed? (util/collapsed? current-block)
+          next-block (when-let [e (db-model/get-next (db/get-db repo) (:db/id current-block))]
+                       (db/pull (:db/id e)))
+          next-block-right (when next-block (db-async/<get-right-sibling repo (:db/id next-block)))
+          db-based? (config/db-based-graph? repo)]
     (cond
       (nil? next-block)
       nil
@@ -2863,19 +2844,10 @@
   (let [editor (state/get-input)
         pos (some-> editor cursor/pos)
         {:keys [block]} (get-state)]
-    (p/do!
-     (when block
-       (state/set-editor-last-pos! pos)
-       (ui-outliner-tx/transact!
-        {:outliner-op :move-blocks
-         :real-outliner-op :indent-outdent}
-        (save-current-block!)
-        (outliner-core/indent-outdent-blocks! (state/get-current-repo)
-                                              (db/get-db false)
-                                              (block-handler/get-top-level-blocks [block])
-                                              indent?
-                                              {:get-first-block-original block-handler/get-first-block-original
-                                               :logical-outdenting? (state/logical-outdenting?)}))))))
+    (when block
+      (state/set-editor-last-pos! pos)
+      (block-handler/indent-outdent-blocks! [block] indent? save-current-block!))))
+
 
 (defn keydown-tab-handler
   [direction]

src/main/frontend/handler/events.cljs

@@ -75,6 +75,7 @@
             [promesa.core :as p]
             [lambdaisland.glogi :as log]
             [rum.core :as rum]
+            [frontend.rum :as r]
             [frontend.persist-db.browser :as db-browser]
             [frontend.db.rtc.debug-ui :as rtc-debug-ui]
             [frontend.modules.outliner.pipeline :as pipeline]
@@ -189,6 +190,9 @@
     (state/set-state! :sync-graph/init? false)))
 
 (defmethod handle :graph/switch [[_ graph opts]]
+  (state/set-state! :db/async-queries #{})
+  (reset! r/*key->atom {})
+
   (let [^js sqlite @db-browser/*worker]
     (p/let [writes-finished? (when sqlite (.file-writes-finished? sqlite (state/get-current-repo)))
             request-finished? (ldb/request-finished?)]
@@ -405,14 +409,12 @@
       (when (and (not dir-exists?)
               (not util/nfs?))
         (state/pub-event! [:graph/dir-gone dir]))))
-  (p/let [loaded-homepage-files (when-not (config/db-based-graph? repo)
-                                  (fs-watcher/preload-graph-homepage-files!))
-          ;; re-render-root is async and delegated to rum, so we need to wait for main ui to refresh
+  (p/let [;; re-render-root is async and delegated to rum, so we need to wait for main ui to refresh
           _ (js/setTimeout #(mobile/mobile-postinit) 1000)
           ;; FIXME: an ugly implementation for redirecting to page on new window is restored
           _ (repo-handler/graph-ready! repo)
           _ (when-not (config/db-based-graph? repo)
-              (fs-watcher/load-graph-files! repo loaded-homepage-files))]))
+              (fs-watcher/load-graph-files! repo))]))
 
 (defmethod handle :notification/show [[_ {:keys [content status clear?]}]]
   (notification/show! content status clear?))

src/main/frontend/handler/file_based/editor.cljs

@@ -6,8 +6,8 @@
             [frontend.format.block :as block]
             [frontend.db :as db]
             [frontend.format.mldoc :as mldoc]
-            [logseq.outliner.core :as outliner-core]
             [frontend.state :as state]
+            [frontend.modules.outliner.op :as outliner-op]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.util :as util]
             [frontend.util.clock :as clock]
@@ -195,9 +195,7 @@
    {:outliner-op :save-block}
    (doseq [block-id block-ids]
      (when-let [block (set-heading-aux! block-id heading)]
-       (outliner-core/save-block! (state/get-current-repo) (db/get-db false)
-                                  (state/get-date-formatter)
-                                  block)))))
+       (outliner-op/save-block! block)))))
 
 (defn set-blocks-id!
   "Persist block uuid to file if the uuid is valid, and it's not persisted in file.

src/main/frontend/handler/file_based/page_property.cljs

@@ -2,7 +2,7 @@
   "Page property fns for file graphs"
   (:require [clojure.string :as string]
             [frontend.db :as db]
-            [logseq.outliner.core :as outliner-core]
+            [frontend.modules.outliner.op :as outliner-op]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.state :as state]
             [frontend.util :as util]))
@@ -85,4 +85,4 @@
             (ui-outliner-tx/transact!
              {:outliner-op :insert-blocks
               :additional-tx page-properties-tx}
-             (outliner-core/insert-blocks! repo (db/get-db false) block page {:sibling? false}))))))))
+             (outliner-op/insert-blocks! block page {:sibling? false}))))))))

src/main/frontend/handler/file_based/property.cljs

@@ -3,7 +3,7 @@
   (:require [frontend.db :as db]
             [frontend.handler.block :as block-handler]
             [frontend.handler.file-based.property.util :as property-util]
-            [logseq.outliner.core :as outliner-core]
+            [frontend.modules.outliner.op :as outliner-op]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.state :as state]
             [logseq.common.util :as common-util]
@@ -55,9 +55,7 @@
                          :block/properties-order property-ks
                          :block/properties-text-values properties-text-values
                          :block/content content}]
-              (outliner-core/save-block! (state/get-current-repo) (db/get-db false)
-                                         (state/get-date-formatter)
-                                         block))))))
+              (outliner-op/save-block! block))))))
      (let [block-id (ffirst col)
            block-id (if (string? block-id) (uuid block-id) block-id)
            input-pos (or (state/get-edit-pos) :max)]

src/main/frontend/handler/import.cljs

@@ -16,15 +16,17 @@
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.common.util :as common-util]
             [logseq.graph-parser.whiteboard :as gp-whiteboard]
-            [logseq.graph-parser.date-time-util :as date-time-util]
+            [logseq.common.util.date-time :as date-time-util]
             [frontend.handler.page :as page-handler]
             [frontend.handler.editor :as editor]
             [frontend.handler.notification :as notification]
             [frontend.util :as util]
             [clojure.core.async :as async]
+            [cljs.core.async.interop :refer [p->c]]
             [medley.core :as medley]
             [frontend.persist-db :as persist-db]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.db.async :as db-async]))
 
 (defn index-files!
   "Create file structure, then parse into DB (client only)"
@@ -195,9 +197,9 @@
               (async/<! (async/timeout 10))
               (create-page-with-exported-tree! block)
               (recur))
-            (do
-              (editor/set-blocks-id! (db/get-all-referenced-blocks-uuid))
-              (async/offer! imported-chan true)))))
+            (let [result (async/<! (p->c (db-async/<get-all-referenced-blocks-uuid (state/get-current-repo))))]
+                (editor/set-blocks-id! result)
+                (async/offer! imported-chan true)))))
 
       (catch :default e
         (notification/show! (str "Error happens when importing:\n" e) :error)

src/main/frontend/handler/page.cljs

@@ -41,11 +41,11 @@
             [frontend.db.conn :as conn]
             [logseq.db :as ldb]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
-            [logseq.outliner.core :as outliner-core]))
+            [frontend.modules.outliner.op :as outliner-op]))
 
 (def create! page-common-handler/create!)
 (def <create! page-common-handler/<create!)
-(def delete! page-common-handler/delete!)
+(def <delete! page-common-handler/<delete!)
 
 (defn <unfavorite-page!
   [page-name]
@@ -139,8 +139,7 @@
 
 (defn <reorder-favorites!
   [favorites]
-  (let [repo (state/get-current-repo)
-        conn (conn/get-db false)]
+  (let [conn (conn/get-db false)]
     (when-let [favorites-page-entity (d/entity @conn [:block/name page-common-handler/favorites-page-name])]
       (let [favorite-page-block-db-id-coll
             (keep (fn [page-name]
@@ -150,13 +149,12 @@
             current-blocks (ldb/sort-by-left (ldb/get-page-blocks @conn page-common-handler/favorites-page-name {})
                                              favorites-page-entity)]
         (p/do!
-          (ui-outliner-tx/transact!
-              {}
-              (doseq [[page-block-db-id block] (zipmap favorite-page-block-db-id-coll current-blocks)]
-                (when (not= page-block-db-id (:db/id (:block/link block)))
-                  (outliner-core/save-block! repo conn (state/get-date-formatter)
-                                             (assoc block :block/link page-block-db-id)))))
-          (state/update-favorites-updated!))))))
+         (ui-outliner-tx/transact!
+          {}
+          (doseq [[page-block-db-id block] (zipmap favorite-page-block-db-id-coll current-blocks)]
+            (when (not= page-block-db-id (:db/id (:block/link block)))
+              (outliner-op/save-block! (assoc block :block/link page-block-db-id)))))
+         (state/update-favorites-updated!))))))
 
 (defn has-more-journals?
   []

src/main/frontend/handler/whiteboard.cljs

@@ -204,11 +204,7 @@
   ([name]
    (p/let [uuid (or (and name (parse-uuid name)) (d/squuid))
            name (or name (str uuid))
-           repo (state/get-current-repo)
            _ (db/transact! (get-default-new-whiteboard-tx name uuid))]
-     ;; TODO: check to remove this
-     (state/update-state! [repo :unloaded-pages] (fn [pages] (conj (set pages)
-                                                                   (util/page-name-sanity-lc name))))
      name)))
 
 (defn <create-new-whiteboard-and-redirect!

src/main/frontend/modules/outliner/op.cljs

@@ -0,0 +1,73 @@
+(ns frontend.modules.outliner.op
+  "Build outliner ops"
+  (:require [datascript.impl.entity :as de]))
+
+(def ^:private ^:dynamic *outliner-ops*
+  "Stores outliner ops that are generated by the following calls"
+  nil)
+
+(defn- op-transact!
+  [fn-var & args]
+  {:pre [(var? fn-var)]}
+  (when (nil? *outliner-ops*)
+    (throw (js/Error. (str (:name (meta fn-var)) " is not used in (transact! ...)"))))
+  (let [result (apply @fn-var args)]
+    (conj! *outliner-ops* result)
+    result))
+
+(defn save-block
+  [block]
+  (when-let [block' (if (de/entity? block)
+                      (assoc (.-kv ^js block) :db/id (:db/id block))
+                      block)]
+    [:save-block [block']]))
+
+(defn insert-blocks
+  [blocks target-block opts]
+  (let [id (:db/id target-block)]
+    [:insert-blocks [blocks id opts]]))
+
+(defn delete-blocks
+  [blocks opts]
+  (let [ids (map :db/id blocks)]
+    [:delete-blocks [ids opts]]))
+
+(defn move-blocks
+  [blocks target-block sibling?]
+  (let [ids (map :db/id blocks)
+        target-id (:db/id target-block)]
+    [:move-blocks [ids target-id sibling?]]))
+
+(defn move-blocks-up-down
+  [blocks up?]
+  (let [ids (map :db/id blocks)]
+    [:move-blocks-up-down [ids up?]]))
+
+(defn indent-outdent-blocks
+  [blocks indent? & {:as opts}]
+  (let [ids (map :db/id blocks)]
+    [:indent-outdent-blocks [ids indent? opts]]))
+
+(defn save-block!
+  [block]
+  (op-transact! #'save-block block))
+
+(defn insert-blocks!
+  [blocks target-block opts]
+  (op-transact! #'insert-blocks blocks target-block opts))
+
+(defn delete-blocks!
+  [blocks opts]
+  (op-transact! #'delete-blocks blocks opts))
+
+(defn move-blocks!
+  [blocks target-block sibling?]
+  (op-transact! #'move-blocks blocks target-block sibling?))
+
+(defn move-blocks-up-down!
+  [blocks up?]
+  (op-transact! #'move-blocks-up-down blocks up?))
+
+(defn indent-outdent-blocks!
+  [blocks indent? & {:as opts}]
+  (op-transact! #'indent-outdent-blocks blocks indent? opts))

src/main/frontend/modules/outliner/pipeline.cljs

@@ -8,7 +8,8 @@
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.history :as history]
             [logseq.db :as ldb]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.util :as util]))
 
 (defn store-undo-data!
   [{:keys [tx-meta] :as opts}]
@@ -18,10 +19,6 @@
               (:whiteboard/transact? tx-meta))
       (undo-redo/listen-db-changes! opts))))
 
-(defn- mark-pages-as-loaded!
-  [repo page-names]
-  (state/update-state! [repo :unloaded-pages] #(remove page-names %)))
-
 (defn- get-tx-id
   [tx-report]
   (get-in tx-report [:tempids :db/current-tx]))
@@ -45,52 +42,54 @@
   ;;      :request-id request-id
   ;;      :tx-meta tx-meta
   ;;      :tx-data tx-data)
-  (let [{:keys [from-disk? new-graph? local-tx? undo? redo?]} tx-meta
+  (let [{:keys [from-disk? new-graph? local-tx? undo? redo? initial-pages? end?]} tx-meta
         repo (state/get-current-repo)
         tx-report {:tx-meta tx-meta
-                   :tx-data tx-data}]
-
-    (let [conn (db/get-db repo false)
-          tx-report (d/transact! conn tx-data tx-meta)]
-      (when local-tx?
-        (let [tx-id (get-tx-id tx-report)]
-          (store-undo-data! (assoc opts :tx-id tx-id))))
-      (when-not (or undo? redo?)
-        (update-current-tx-editor-cursor! tx-report)))
-
-    (let [new-datoms (filter (fn [datom]
-                               (and
-                                (= :block/uuid (:a datom))
-                                (true? (:added datom)))) tx-data)]
-      (when (seq new-datoms)
-        (state/set-state! :editor/new-created-blocks (set (map :v new-datoms)))))
-
-    (let [pages (set (keep #(when (= :block/name (:a %)) (:v %)) tx-data))]
-      (when (seq pages)
-        (mark-pages-as-loaded! repo pages)))
-
-    (if (or from-disk? new-graph?)
+                   :tx-data tx-data}
+        conn (db/get-db repo false)]
+    (if initial-pages?
       (do
-        (react/clear-query-state!)
-        (ui-handler/re-render-root!))
-      (when-not (:graph/importing @state/state)
-        (react/refresh! repo tx-report affected-keys)
-
-        (when-let [state (:ui/restore-cursor-state @state/state)]
-          (when (or undo? redo?)
-            (restore-cursor-and-app-state! state undo?)
-            (state/set-state! :ui/restore-cursor-state nil)))
-
-        (state/set-state! :editor/start-pos nil)
-
-        (when (and state/lsp-enabled?
-                   (seq blocks)
-                   (<= (count blocks) 1000))
-          (state/pub-event! [:plugin/hook-db-tx
-                             {:blocks  blocks
-                              :deleted-block-uuids deleted-block-uuids
-                              :tx-data (:tx-data tx-report)
-                              :tx-meta (:tx-meta tx-report)}]))))
+        (util/profile "transact initial-pages" (d/transact! conn tx-data tx-meta))
+        (when end?
+          (state/pub-event! [:init/commands])
+          (ui-handler/re-render-root!)))
+      (do
+        (let [tx-report (d/transact! conn tx-data tx-meta)]
+          (when local-tx?
+            (let [tx-id (get-tx-id tx-report)]
+              (store-undo-data! (assoc opts :tx-id tx-id))))
+          (when-not (or undo? redo?)
+            (update-current-tx-editor-cursor! tx-report)))
+
+        (let [new-datoms (filter (fn [datom]
+                                   (and
+                                    (= :block/uuid (:a datom))
+                                    (true? (:added datom)))) tx-data)]
+          (when (seq new-datoms)
+            (state/set-state! :editor/new-created-blocks (set (map :v new-datoms)))))
+
+        (if (or from-disk? new-graph?)
+          (do
+            (react/clear-query-state!)
+            (ui-handler/re-render-root!))
+          (when-not (:graph/importing @state/state)
+            (react/refresh! repo tx-report affected-keys)
+
+            (when-let [state (:ui/restore-cursor-state @state/state)]
+              (when (or undo? redo?)
+                (restore-cursor-and-app-state! state undo?)
+                (state/set-state! :ui/restore-cursor-state nil)))
+
+            (state/set-state! :editor/start-pos nil)
+
+            (when (and state/lsp-enabled?
+                       (seq blocks)
+                       (<= (count blocks) 1000))
+              (state/pub-event! [:plugin/hook-db-tx
+                                 {:blocks  blocks
+                                  :deleted-block-uuids deleted-block-uuids
+                                  :tx-data (:tx-data tx-report)
+                                  :tx-meta (:tx-meta tx-report)}]))))))
 
     (when (= (:outliner-op tx-meta) :delete-page)
       (state/pub-event! [:page/deleted repo (:deleted-page tx-meta) (:file-path tx-meta) tx-meta]))
@@ -107,6 +106,6 @@
 
     (when request-id
       (when-let [deferred (ldb/get-deferred-response request-id)]
-        (p/resolve! deferred {:tx-meta tx-meta
-                              :tx-data tx-data})
-        (swap! ldb/*request-id->response dissoc request-id)))))
+        (when (p/promise? deferred)
+          (p/resolve! deferred {:tx-meta tx-meta :tx-data tx-data})))
+      (swap! ldb/*request-id->response dissoc request-id))))

src/main/frontend/modules/outliner/ui.cljc

@@ -1,28 +1,32 @@
 (ns frontend.modules.outliner.ui
-  #?(:cljs (:require-macros [logseq.outliner.transaction]))
   #?(:cljs (:require-macros [frontend.modules.outliner.ui]))
   #?(:cljs (:require [frontend.state :as state]
-                     [frontend.config :as config]
-                     [frontend.db :as db])))
-
-#?(:cljs
-   (do
-     (defn unlinked-graph?
-       []
-       (let [repo (state/get-current-repo)]
-         (contains? (:file/unlinked-dirs @state/state)
-                    (config/get-repo-dir repo))))
-
-     (def set-state-fn state/set-state!)))
+                     [frontend.db :as db]
+                     [logseq.outliner.op])))
 
 (defmacro transact!
   [opts & body]
-  `(when (db/request-finished?)
-     (let [transact-opts# {:repo (state/get-current-repo)
-                           :conn (db/get-db false)
-                           :unlinked-graph? frontend.modules.outliner.ui/unlinked-graph?
-                           :set-state-fn frontend.modules.outliner.ui/set-state-fn}]
+  `(let [test?# frontend.util/node-test?]
+     (when (or test?# (db/request-finished?))
        (when (nil? @(:history/tx-before-editor-cursor @state/state))
          (state/set-state! :history/tx-before-editor-cursor (state/get-current-edit-block-and-position)))
-       (logseq.outliner.transaction/transact! (assoc ~opts :transact-opts transact-opts#)
-                                              ~@body))))
+       (let [ops# frontend.modules.outliner.op/*outliner-ops*]
+         (if ops#
+           (do ~@body)                    ; nested transact!
+           (binding [frontend.modules.outliner.op/*outliner-ops* (transient [])]
+             ~@body
+             (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*)
+                   worker# @state/*db-worker]
+               (if (and test?# (seq r#))
+                 (logseq.outliner.op/apply-ops! (state/get-current-repo)
+                                                (db/get-db false)
+                                                r#
+                                                (state/get-date-formatter)
+                                                ~opts)
+                 (when (and worker# (seq r#))
+                   (let [request-id# (state/get-worker-next-request-id)
+                         response# (.apply-outliner-ops ^Object worker# (state/get-current-repo)
+                                                        (pr-str r#)
+                                                        (pr-str (assoc ~opts :request-id request-id#)))]
+                     (state/add-worker-request! request-id# :outliner-tx)
+                     response#))))))))))

src/main/frontend/persist_db/browser.cljs

@@ -15,7 +15,7 @@
             [logseq.db :as ldb]
             [frontend.date :as date]))
 
-(defonce *worker (atom nil))
+(defonce *worker state/*db-worker)
 
 (defn- ask-persist-permission!
   []

src/main/frontend/rum.cljs

@@ -155,3 +155,12 @@
          #(js/document.removeEventListener event listener capture?)))
      [ref])
     set-ref))
+
+(defonce *key->atom (atom {}))
+(defn cached-derived-atom
+  "Make sure to return the same atom if `key` is the same."
+  [ref key f]
+  (or (get @*key->atom key)
+      (let [a (rum/derived-atom [ref] key f)]
+        (swap! *key->atom assoc key a)
+        a)))

src/main/frontend/search.cljs

@@ -14,7 +14,11 @@
             [frontend.config :as config]
             [logseq.db.frontend.property :as db-property]
             [frontend.handler.file-based.property.util :as property-util]
-            [cljs-bean.core :as bean]))
+            [cljs-bean.core :as bean]
+            [frontend.db :as db]
+            [frontend.db.model :as db-model]
+            [frontend.db.utils :as db-utils]
+            [clojure.edn :as edn]))
 
 (def fuzzy-search fuzzy/fuzzy-search)
 
@@ -128,3 +132,22 @@
   [repo data]
   (when-let [engine (get-engine repo)]
     (protocol/transact-blocks! engine data)))
+
+(defn get-page-unlinked-refs
+  "Get matched result from search first, and then filter by worker db"
+  [page]
+  (when-let [repo (state/get-current-repo)]
+    (p/let [page-name (util/safe-page-name-sanity-lc page)
+            page (db/entity [:block/name page-name])
+            alias-names (conj (set (map util/safe-page-name-sanity-lc
+                                        (db/get-page-alias-names repo page-name))) page-name)
+            q (string/join " " alias-names)
+            result (block-search repo q {:limit 100})
+            eids (map (fn [b] [:block/uuid (:block/uuid b)]) result)
+            result (when (seq eids)
+                     (.get-page-unlinked-refs ^Object @state/*db-worker repo (:db/id page) (pr-str eids)))
+            result' (when result (edn/read-string result))]
+      (when result' (db/transact! repo result'))
+      (some->> result'
+               db-model/sort-by-left-recursive
+               db-utils/group-by-page))))

src/main/frontend/state.cljs

@@ -15,15 +15,19 @@
             [goog.dom :as gdom]
             [goog.object :as gobj]
             [logseq.common.config :as common-config]
+            [logseq.db :as ldb]
             [medley.core :as medley]
             [promesa.core :as p]
-            [rum.core :as rum]))
+            [rum.core :as rum]
+            [frontend.rum :as r]))
 
 (defonce *profile-state
   (atom {}))
 
 (defonce *editor-editing-ref (atom nil))
 
+(defonce *db-worker (atom nil))
+
 ;; Stores main application state
 (defonce ^:large-vars/data-var state
   (let [document-mode? (or (storage/get :document/mode?) false)
@@ -308,8 +312,8 @@
       :system/info                           {}
       ;; Whether block is selected
       :ui/select-query-cache                 (atom {})
-
-      :favorites/updated?                    (atom 0)})))
+      :favorites/updated?                    (atom 0)
+      :db/async-queries                      (atom #{})})))
 
 ;; Block ast state
 ;; ===============
@@ -694,8 +698,8 @@ Similar to re-frame subscriptions"
   (let [*cache (:ui/select-query-cache @state)
         keys [::select-block block-uuid]
         atom (or (get @*cache keys)
-                 (let [result (rum/derived-atom
-                               [(:selection/blocks @state)]
+                 (let [result (r/cached-derived-atom
+                               (:selection/blocks @state)
                                keys
                                (fn [s]
                                  (contains? (set (get-selected-block-ids s)) block-uuid)))]
@@ -2309,19 +2313,12 @@ Similar to re-frame subscriptions"
   []
   (storage/remove :user-groups))
 
-(defn sub-block-unloaded?
-  [repo block-uuid]
-  (rum/react
-   (rum/derived-atom [(rum/cursor-in state [repo :restore/unloaded-blocks])] [::block-unloaded repo block-uuid]
-     (fn [s]
-       (contains? s (str block-uuid))))))
-
-(defn sub-page-unloaded?
-  [repo page-name]
+(defn sub-async-query-loading
+  [k]
+  (assert (some? k))
   (rum/react
-   (rum/derived-atom [(rum/cursor-in state [repo :unloaded-pages])] [::page-unloaded repo page-name]
-                     (fn [s]
-                       (contains? s page-name)))))
+   (r/cached-derived-atom (:db/async-queries @state) [(get-current-repo) ::async-query (str k)]
+                          (fn [s] (contains? s (str k))))))
 
 (defn get-color-accent []
   (get @state :ui/radix-color))
@@ -2372,3 +2369,6 @@ Similar to re-frame subscriptions"
 (defn update-favorites-updated!
   []
   (update-state! :favorites/updated? inc))
+
+(def get-worker-next-request-id ldb/get-next-request-id)
+(def add-worker-request! ldb/add-request!)

src/main/frontend/worker/rtc/core.cljs

@@ -394,7 +394,7 @@
   [repo conn remove-page-ops]
   (doseq [op remove-page-ops]
     (when-let [page-name (:block/name (d/entity @conn [:block/uuid (:block-uuid op)]))]
-      (worker-page/delete! repo conn page-name nil {:redirect-to-home? false :persist-op? false}))))
+      (worker-page/delete! repo conn page-name nil {:persist-op? false}))))
 
 (defn filter-remote-data-by-local-unpushed-ops
   "when remote-data request client to move/update/remove/... blocks,

src/main/frontend/worker/state.cljs

@@ -1,6 +1,7 @@
 (ns frontend.worker.state
   "State hub for worker"
-  (:require [logseq.common.util :as common-util]))
+  (:require [logseq.common.util :as common-util]
+            [logseq.common.config :as common-config]))
 
 (defonce *state (atom {:worker/object nil
 
@@ -79,3 +80,7 @@
 (defn get-worker-object
   []
   (:worker/object @*state))
+
+(defn get-date-formatter
+  [repo]
+  (common-config/get-date-formatter (get-config repo)))

src/main/logseq/api.cljs

@@ -7,7 +7,6 @@
             [logseq.sdk.ui :as sdk-ui]
             [logseq.sdk.assets :as sdk-assets]
             [clojure.string :as string]
-            [datascript.core :as d]
             [electron.ipc :as ipc]
             [frontend.commands :as commands]
             [frontend.components.plugins :as plugins]
@@ -53,6 +52,17 @@
 
 ;; Alert: this namespace shouldn't invoke any reactive queries
 
+(defn- <pull-block
+  [id-or-name]
+  (when id-or-name
+    (let [eid (cond
+                (uuid? id-or-name) [:block/uuid id-or-name]
+                (and (vector? id-or-name) (= (count id-or-name) 2)) id-or-name
+                (number? id-or-name) id-or-name
+                (and (string? id-or-name) (util/uuid-string? id-or-name)) [:block/uuid (uuid id-or-name)]
+                :else [:block/name (util/page-name-sanity-lc id-or-name)])]
+      (db-async/<pull (state/get-current-repo) eid))))
+
 ;; helpers
 (defn ^:export install-plugin-hook
   [pid hook ^js opts]
@@ -142,7 +152,6 @@
     (when-let [repo (state/get-current-repo)]
       (p/let [templates (db-async/<get-all-templates repo)]
         (some-> templates
-                (update-vals db/pull)
                 (sdk-utils/normalize-keyword-for-json)
                 (bean/->js))))))
 
@@ -538,16 +547,21 @@
 (def ^:export get_current_page
   (fn []
     (when-let [page (state/get-current-page)]
-      (when-let [page (db-model/get-page page)]
+      (p/let [page (<pull-block page)]
         (bean/->js (sdk-utils/normalize-keyword-for-json (db-utils/pull (:db/id page))))))))
 
 (def ^:export get_page
   (fn [id-or-page-name]
-    (when-let [page (cond
-                      (number? id-or-page-name) (db-utils/pull id-or-page-name)
-                      (string? id-or-page-name) (db-model/get-page id-or-page-name))]
-      (when-not (contains? page :block/left)
-        (bean/->js (sdk-utils/normalize-keyword-for-json (db-utils/pull (:db/id page))))))))
+    (p/let [page (db-async/<pull (state/get-current-repo)
+                                 (cond
+                                   (number? id-or-page-name)
+                                   id-or-page-name
+                                   (util/uuid-string? id-or-page-name)
+                                   [:block/uuid (uuid id-or-page-name)]
+                                   :else
+                                   [:block/name (util/page-name-sanity-lc id-or-page-name)]))]
+      (when (:block/name page)
+        (bean/->js (sdk-utils/normalize-keyword-for-json page))))))
 
 (def ^:export get_all_pages
   (fn [repo]
@@ -558,7 +572,7 @@
   (fn [name ^js properties ^js opts]
     (let [properties (bean/->clj properties)
           {:keys [redirect createFirstBlock format journal]} (bean/->clj opts)]
-      (p/let [page (db-model/get-page name)
+      (p/let [page (<pull-block name)
               new-page (when-not page
                          (page-handler/<create!
                           name
@@ -575,7 +589,7 @@
 
 (def ^:export delete_page
   (fn [name]
-    (p/create (fn [ok] (page-handler/delete! name ok)))))
+    (page-handler/<delete! name nil)))
 
 (def ^:export rename_page
   page-handler/rename!)
@@ -602,100 +616,124 @@
         (let [{:keys [pos] :or {pos :max}} (bean/->clj opts)]
           (editor-handler/edit-block! block pos block-uuid))))))
 
+;; TODO: perf improvement, some operations such as delete-block doesn't need to load the full page
+;; instead, the db worker should provide those calls
+(defn- <ensure-page-loaded
+  [block-uuid-or-page-name]
+  (p/let [repo (state/get-current-repo)
+          result (db-async/<get-block repo (str block-uuid-or-page-name))
+          block (if (:block result) (:block result) result)
+          _ (when-let [page-id (:db/id (:block/page block))]
+              (when-let [page-uuid (:block/uuid (db/entity page-id))]
+                (db-async/<get-block repo page-uuid)))]
+    block))
+
 (def ^:export insert_block
   (fn [block-uuid-or-page-name content ^js opts]
     (when (string/blank? block-uuid-or-page-name)
       (throw (js/Error. "Page title or block UUID shouldn't be empty.")))
-    (p/let [{:keys [before sibling focus customUUID properties autoOrderedList]} (bean/->clj opts)
-            [page-name block-uuid] (if (util/uuid-string? block-uuid-or-page-name)
-                                     [nil (uuid block-uuid-or-page-name)]
-                                     [block-uuid-or-page-name nil])
-            page-name              (when page-name (util/page-name-sanity-lc page-name))
-            _                      (when (and page-name (not (db/entity [:block/name page-name])))
-                                     (page-handler/<create! block-uuid-or-page-name {:create-first-block? false}))
-            custom-uuid            (or customUUID (:id properties))
-            custom-uuid            (when custom-uuid (sdk-utils/uuid-or-throw-error custom-uuid))
-            edit-block?            (if (nil? focus) true focus)
-            _                      (when (and custom-uuid (db-model/query-block-by-uuid custom-uuid))
-                                     (throw (js/Error.
-                                             (util/format "Custom block UUID already exists (%s)." custom-uuid))))
-            block-uuid'            (if (and (not sibling) before block-uuid)
-                                     (let [block       (db/entity [:block/uuid block-uuid])
-                                           first-child (db-model/get-by-parent-&-left (db/get-db)
-                                                                                      (:db/id block)
-                                                                                      (:db/id block))]
-                                       (if first-child
-                                         (:block/uuid first-child)
-                                         block-uuid))
-                                     block-uuid)
-            insert-at-first-child? (not= block-uuid' block-uuid)
-            [sibling? before?] (if insert-at-first-child?
-                                 [true true]
-                                 [sibling before])
-            before?                (if (and (false? sibling?) before? (not insert-at-first-child?))
-                                     false
-                                     before?)
-            new-block              (editor-handler/api-insert-new-block!
-                                    content
-                                    {:block-uuid  block-uuid'
-                                     :sibling?    sibling?
-                                     :before?     before?
-                                     :edit-block? edit-block?
-                                     :page        page-name
-                                     :custom-uuid custom-uuid
-                                     :ordered-list? (if (boolean? autoOrderedList) autoOrderedList false)
-                                     :properties  (merge properties
-                                                         (when custom-uuid {:id custom-uuid}))})]
-      (bean/->js (sdk-utils/normalize-keyword-for-json new-block)))))
+    (p/let [block? (util/uuid-string? (str block-uuid-or-page-name))
+            block (<pull-block (str block-uuid-or-page-name))]
+      (if (and block? (not block))
+        (throw (js/Error. "Block not exists"))
+        (p/let [{:keys [before sibling focus customUUID properties autoOrderedList]} (bean/->clj opts)
+                [page-name block-uuid] (if (util/uuid-string? block-uuid-or-page-name)
+                                         [nil (uuid block-uuid-or-page-name)]
+                                         [block-uuid-or-page-name nil])
+                page-name              (when page-name (util/page-name-sanity-lc page-name))
+                _                      (when (and page-name (not (db/entity [:block/name page-name])))
+                                         (page-handler/<create! block-uuid-or-page-name {:create-first-block? false}))
+                custom-uuid            (or customUUID (:id properties))
+                custom-uuid            (when custom-uuid (sdk-utils/uuid-or-throw-error custom-uuid))
+                edit-block?            (if (nil? focus) true focus)
+                _                      (when (and custom-uuid (db-model/query-block-by-uuid custom-uuid))
+                                         (throw (js/Error.
+                                                 (util/format "Custom block UUID already exists (%s)." custom-uuid))))
+                block-uuid'            (if (and (not sibling) before block-uuid)
+                                         (let [block       (db/entity [:block/uuid block-uuid])
+                                               first-child (db-model/get-by-parent-&-left (db/get-db)
+                                                                                          (:db/id block)
+                                                                                          (:db/id block))]
+                                           (if first-child
+                                             (:block/uuid first-child)
+                                             block-uuid))
+                                         block-uuid)
+                insert-at-first-child? (not= block-uuid' block-uuid)
+                [sibling? before?] (if insert-at-first-child?
+                                     [true true]
+                                     [sibling before])
+                before?                (if (and (false? sibling?) before? (not insert-at-first-child?))
+                                         false
+                                         before?)
+                new-block              (editor-handler/api-insert-new-block!
+                                        content
+                                        {:block-uuid  block-uuid'
+                                         :sibling?    sibling?
+                                         :before?     before?
+                                         :edit-block? edit-block?
+                                         :page        page-name
+                                         :custom-uuid custom-uuid
+                                         :ordered-list? (if (boolean? autoOrderedList) autoOrderedList false)
+                                         :properties  (merge properties
+                                                             (when custom-uuid {:id custom-uuid}))})]
+          (bean/->js (sdk-utils/normalize-keyword-for-json new-block)))))))
 
 (def ^:export insert_batch_block
   (fn [block-uuid ^js batch-blocks ^js opts]
-    (when-let [block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error block-uuid))]
-      (when-let [bb (bean/->clj batch-blocks)]
-        (let [bb (if-not (vector? bb) (vector bb) bb)
-              {:keys [sibling keepUUID before]} (bean/->clj opts)
-              keep-uuid? (or keepUUID false)
-              _ (when keep-uuid? (doseq
-                                  [block (outliner-core/tree-vec-flatten bb :children)]
-                                   (let [uuid (:id (:properties block))]
-                                     (when (and uuid (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error uuid)))
-                                       (throw (js/Error.
-                                                (util/format "Custom block UUID already exists (%s)." uuid)))))))
-              block (if (and before sibling)
-                      (db/pull (:db/id (:block/left block))) block)
-              _ (editor-handler/insert-block-tree-after-target
-                  (:db/id block) sibling bb (:block/format block) keep-uuid?)]
-          nil)))))
+    (p/let [block (<ensure-page-loaded block-uuid)]
+      (when block
+        (when-let [bb (bean/->clj batch-blocks)]
+          (let [bb (if-not (vector? bb) (vector bb) bb)
+                {:keys [sibling keepUUID before]} (bean/->clj opts)
+                keep-uuid? (or keepUUID false)
+                _ (when keep-uuid? (doseq
+                                    [block (outliner-core/tree-vec-flatten bb :children)]
+                                     (let [uuid (:id (:properties block))]
+                                       (when (and uuid (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error uuid)))
+                                         (throw (js/Error.
+                                                 (util/format "Custom block UUID already exists (%s)." uuid)))))))
+                block (if (and before sibling)
+                        (db/pull (:db/id (:block/left block))) block)
+                _ (editor-handler/insert-block-tree-after-target
+                   (:db/id block) sibling bb (:block/format block) keep-uuid?)]
+            nil))))))
 
 (def ^:export remove_block
   (fn [block-uuid ^js _opts]
-    (let [repo            (state/get-current-repo)]
+    (p/let [repo            (state/get-current-repo)
+            _ (<pull-block  block-uuid)]
       (editor-handler/delete-block-aux!
        {:block/uuid (sdk-utils/uuid-or-throw-error block-uuid) :repo repo} true))))
 
 (def ^:export update_block
   (fn [block-uuid content ^js opts]
-    (let [repo (state/get-current-repo)]
+    (p/let [repo (state/get-current-repo)
+            _ (<pull-block block-uuid)]
       (editor-handler/save-block! repo
-        (sdk-utils/uuid-or-throw-error block-uuid) content (bean/->clj opts)))))
+                                  (sdk-utils/uuid-or-throw-error block-uuid) content (bean/->clj opts)))))
 
 (def ^:export move_block
   (fn [src-block-uuid target-block-uuid ^js opts]
-    (let [{:keys [before children]} (bean/->clj opts)
-          move-to      (cond
-                         (boolean before)
-                         :top
-
-                         (boolean children)
-                         :nested
-
-                         :else
-                         nil)
-          src-block    (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error src-block-uuid))
-          target-block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error target-block-uuid))]
-      (editor-dnd-handler/move-blocks nil [src-block] target-block nil move-to))))
-
-(def ^:export get_block api-block/get_block)
+    (p/let [_ (<pull-block src-block-uuid)
+            _ (<pull-block target-block-uuid)]
+      (let [{:keys [before children]} (bean/->clj opts)
+           move-to      (cond
+                          (boolean before)
+                          :top
+
+                          (boolean children)
+                          :nested
+
+                          :else
+                          nil)
+           src-block    (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error src-block-uuid))
+           target-block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error target-block-uuid))]
+       (editor-dnd-handler/move-blocks nil [src-block] target-block nil move-to)))))
+
+(def ^:export get_block
+  (fn [id ^js opts]
+    (p/let [_ (db-async/<get-block (state/get-current-repo) id)]
+      (api-block/get_block id opts))))
 
 (def ^:export get_current_block
   (fn [^js opts]
@@ -709,21 +747,27 @@
 
 (def ^:export get_previous_sibling_block
   (fn [block-uuid]
-    (when-let [block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error block-uuid))]
-      (let [{:block/keys [parent left]} block
-            block (when-not (= parent left) (db-utils/pull (:db/id left)))]
-        (and block (bean/->js (sdk-utils/normalize-keyword-for-json block)))))))
+    (p/let [id (sdk-utils/uuid-or-throw-error block-uuid)
+            block (<pull-block id)]
+      (when block
+       (p/let [{:block/keys [parent left]} block
+               block (when-not (= parent left) (<pull-block (:db/id left)))]
+         (when block
+           (bean/->js (sdk-utils/normalize-keyword-for-json block))))))))
 
 (def ^:export get_next_sibling_block
   (fn [block-uuid]
-    (when-let [block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error block-uuid))]
-      (when-let [right-sibling (outliner-core/get-right-sibling (db/get-db) (:db/id block))]
-        (let [block (db/pull (:db/id right-sibling))]
-          (bean/->js (sdk-utils/normalize-keyword-for-json block)))))))
+    (p/let [id (sdk-utils/uuid-or-throw-error block-uuid)
+            block (<pull-block id)]
+      (when block
+        (p/let [sibling (db-async/<get-right-sibling (state/get-current-repo) (:db/id block))]
+          (when sibling
+            (bean/->js (sdk-utils/normalize-keyword-for-json sibling))))))))
 
 (def ^:export set_block_collapsed
   (fn [block-uuid ^js opts]
-    (let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid)]
+    (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid)
+            _ (db-async/<get-block (state/get-current-repo) block-uuid :children? false)]
       (when-let [block (db-model/get-block-by-uuid block-uuid)]
         (let [opts (bean/->clj opts)
               opts (if (or (string? opts) (boolean? opts)) {:flag opts} opts)
@@ -732,34 +776,42 @@
                      (not (util/collapsed? block))
                      (boolean flag))]
           (if flag (editor-handler/collapse-block! block-uuid)
-                   (editor-handler/expand-block! block-uuid))
+              (editor-handler/expand-block! block-uuid))
           nil)))))
 
 (def ^:export upsert_block_property
   (fn [block-uuid key value]
-    (property-handler/set-block-property!
-     (state/get-current-repo)
-     (sdk-utils/uuid-or-throw-error block-uuid) key value)))
+    (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid)
+            _ (db-async/<get-block (state/get-current-repo) block-uuid :children? false)]
+      (property-handler/set-block-property!
+       (state/get-current-repo)
+       block-uuid key value))))
 
 (def ^:export remove_block_property
   (fn [block-uuid key]
-    (property-handler/remove-block-property!
-     (state/get-current-repo)
-     (sdk-utils/uuid-or-throw-error block-uuid) key)))
+    (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid)
+            _ (db-async/<get-block (state/get-current-repo) block-uuid :children? false)]
+      (property-handler/remove-block-property!
+      (state/get-current-repo)
+      block-uuid key))))
 
 (def ^:export get_block_property
   (fn [block-uuid key]
-    (when-let [block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error block-uuid))]
-      (let [property-id (pu/get-pid key)]
-        (get (:block/properties block) (if (string? property-id) (keyword property-id) property-id))))))
+    (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid)
+            _ (db-async/<get-block (state/get-current-repo) block-uuid :children? false)]
+      (when-let [block (db-model/query-block-by-uuid block-uuid)]
+        (let [property-id (pu/get-pid key)]
+          (get (:block/properties block) (if (string? property-id) (keyword property-id) property-id)))))))
 
 (def ^:export get_block_properties
   (fn [block-uuid]
-    (when-let [block (db-model/query-block-by-uuid (sdk-utils/uuid-or-throw-error block-uuid))]
-      (let [properties (if (config/db-based-graph? (state/get-current-repo))
-                         (db-pu/readable-properties (:block/properties block))
-                         (:block/properties block))]
-        (bean/->js (sdk-utils/normalize-keyword-for-json properties))))))
+    (p/let [block-uuid (sdk-utils/uuid-or-throw-error block-uuid)
+            _ (db-async/<get-block (state/get-current-repo) block-uuid :children? false)]
+      (when-let [block (db-model/query-block-by-uuid block-uuid)]
+        (let [properties (if (config/db-based-graph? (state/get-current-repo))
+                           (db-pu/readable-properties (:block/properties block))
+                           (:block/properties block))]
+          (bean/->js (sdk-utils/normalize-keyword-for-json properties)))))))
 
 (def ^:export get_current_page_blocks_tree
   (fn []
@@ -772,21 +824,25 @@
 
 (def ^:export get_page_blocks_tree
   (fn [id-or-page-name]
-    (when-let [page-name (:block/name (db-model/get-page id-or-page-name))]
-      (let [blocks (db-model/get-page-blocks-no-cache page-name)
-            blocks (outliner-tree/blocks->vec-tree blocks page-name)
-            blocks (sdk-utils/normalize-keyword-for-json blocks)]
-        (bean/->js blocks)))))
+    (p/let [_ (<ensure-page-loaded id-or-page-name)]
+      (when-let [page-name (:block/name (db-model/get-page id-or-page-name))]
+        (let [blocks (db-model/get-page-blocks-no-cache page-name)
+              blocks (outliner-tree/blocks->vec-tree blocks page-name)
+              blocks (sdk-utils/normalize-keyword-for-json blocks)]
+          (bean/->js blocks))))))
 
 (defn ^:export get_page_linked_references
   [page-name-or-uuid]
-  (when-let [page (and page-name-or-uuid (db-model/get-page page-name-or-uuid))]
-    (let [page-name  (:block/name page)
+  (p/let [repo (state/get-current-repo)
+          block (db-async/<get-block repo page-name-or-uuid :children? false)
+          ;; load refs to db
+          _ (when-let [id (:db/id block)] (db-async/<get-block-refs repo id))
+          page-name (:block/name block)
           ref-blocks (if page-name
                        (db-model/get-page-referenced-blocks-full page-name)
-                       (db-model/get-block-referenced-blocks (:block/uuid page)))
+                       (db-model/get-block-referenced-blocks (:block/uuid block)))
           ref-blocks (and (seq ref-blocks) (into [] ref-blocks))]
-      (bean/->js (sdk-utils/normalize-keyword-for-json ref-blocks)))))
+    (bean/->js (sdk-utils/normalize-keyword-for-json ref-blocks))))
 
 (defn ^:export get_pages_from_namespace
   [ns]
@@ -800,16 +856,6 @@
     (when-let [pages (db-model/get-namespace-hierarchy repo ns)]
       (bean/->js (sdk-utils/normalize-keyword-for-json pages)))))
 
-(defn first-child-of-block
-  [block]
-  (when-let [children (:block/_parent block)]
-    (first (db-model/sort-by-left children block))))
-
-(defn second-child-of-block
-  [block]
-  (when-let [children (:block/_parent block)]
-    (second (db-model/sort-by-left children block))))
-
 (defn last-child-of-block
   [block]
   (when-let [children (:block/_parent block)]
@@ -817,36 +863,32 @@
 
 (defn ^:export prepend_block_in_page
   [uuid-or-page-name content ^js opts]
-  (p/let [page?           (not (util/uuid-string? uuid-or-page-name))
+  (p/let [_               (<pull-block uuid-or-page-name)
+          page?           (not (util/uuid-string? uuid-or-page-name))
           page-not-exist? (and page? (nil? (db-model/get-page uuid-or-page-name)))
           _               (and page-not-exist? (page-handler/<create! uuid-or-page-name
                                                                       {:redirect?           false
-                                                                       :create-first-block? true
+                                                                       :create-first-block? false
                                                                        :format              (state/get-preferred-format)}))]
     (when-let [block (db-model/get-page uuid-or-page-name)]
-      (let [block'   (if page? (second-child-of-block block) (first-child-of-block block))
-            sibling? (and page? (not (nil? block')))
-            opts     (bean/->clj opts)
-            opts     (merge opts {:sibling sibling? :before sibling?})
-            src      (if sibling? (str (:block/uuid block')) uuid-or-page-name)]
-        (insert_block src content (bean/->js opts))))))
+      (let [opts     (bean/->clj opts)
+            target   (str (:block/uuid block))]
+        (insert_block target content (bean/->js opts))))))
 
 (defn ^:export append_block_in_page
   [uuid-or-page-name content ^js opts]
-  (p/let [page?           (not (util/uuid-string? uuid-or-page-name))
+  (p/let [_               (<ensure-page-loaded uuid-or-page-name)
+          page?           (not (util/uuid-string? uuid-or-page-name))
           page-not-exist? (and page? (nil? (db-model/get-page uuid-or-page-name)))
           _               (and page-not-exist? (page-handler/<create! uuid-or-page-name
                                                                       {:redirect?           false
-                                                                       :create-first-block? true
+                                                                       :create-first-block? false
                                                                        :format              (state/get-preferred-format)}))]
     (when-let [block (db-model/get-page uuid-or-page-name)]
-      (let [block'   (last-child-of-block block)
-            sibling? (not (nil? block'))
+      (let [block    (or (last-child-of-block block) block)
             opts     (bean/->clj opts)
-            opts     (merge opts {:sibling sibling?}
-                            (when sibling? {:before false}))
-            src      (if sibling? (str (:block/uuid block')) uuid-or-page-name)]
-        (insert_block src content (bean/->js opts))))))
+            target   (str (:block/uuid block))]
+        (insert_block target content (bean/->js opts))))))
 
 ;; plugins
 (defn ^:export validate_external_plugins [urls]
@@ -863,34 +905,36 @@
 (defn ^:export q
   [query-string]
   (when-let [repo (state/get-current-repo)]
-    (when-let [result (query-dsl/query repo query-string
-                        {:disable-reactive? true})]
+    (p/let [result (query-dsl/query repo query-string
+                                    {:disable-reactive? true
+                                     :return-promise? true})]
       (bean/->js (sdk-utils/normalize-keyword-for-json (flatten @result))))))
 
 (defn ^:export datascript_query
   [query & inputs]
   (when-let [repo (state/get-current-repo)]
     (when-let [db (db/get-db repo)]
-      (let [query           (cljs.reader/read-string query)
-            resolved-inputs (map #(cond
-                                    (string? %)
-                                    (some->> % (cljs.reader/read-string) (query-react/resolve-input db))
-
-                                    (fn? %)
-                                    (fn [& args]
-                                      (.apply % nil (clj->js (mapv bean/->js args))))
-
-                                    :else %)
-                                 inputs)
-            result          (apply d/q query db resolved-inputs)]
+      (p/let [query           (cljs.reader/read-string query)
+              resolved-inputs (map #(cond
+                                      (string? %)
+                                      (some->> % (cljs.reader/read-string) (query-react/resolve-input db))
+
+                                      (fn? %)
+                                      (fn [& args]
+                                        (.apply % nil (clj->js (mapv bean/->js args))))
+
+                                      :else %)
+                                   inputs)
+              result          (apply db-async/<q repo (cons query resolved-inputs))]
         (bean/->js (sdk-utils/normalize-keyword-for-json result false))))))
 
 (defn ^:export custom_query
   [query-string]
-  (let [result (let [query (cljs.reader/read-string query-string)]
-                 (db/custom-query {:query query
-                                   :disable-reactive? true}))]
-    (bean/->js (sdk-utils/normalize-keyword-for-json (flatten @result)))))
+  (p/let [result (let [query (cljs.reader/read-string query-string)]
+                   (db/custom-query {:query query
+                                     :disable-reactive? true
+                                     :return-promise? true}))]
+    (bean/->js (sdk-utils/normalize-keyword-for-json (flatten result)))))
 
 (defn ^:export download_graph_db
   []
@@ -963,10 +1007,10 @@
 ;; templates
 (defn ^:export get_template
   [name]
-  (some-> name
-          (db-model/get-template-by-name)
-          (sdk-utils/normalize-keyword-for-json)
-          (bean/->js)))
+  (p/let [block (when name (db-async/<get-template-by-name name))]
+    (some-> block
+            (sdk-utils/normalize-keyword-for-json)
+            (bean/->js))))
 
 (defn ^:export insert_template
   [target-uuid template-name]
@@ -983,20 +1027,21 @@
   [target-uuid template-name ^js opts]
   (when (and template-name (db-model/get-block-by-uuid target-uuid))
     (p/let [{:keys [overwrite]} (bean/->clj opts)
-            exist? (page-handler/<template-exists? template-name)
+            block (db-async/<get-template-by-name template-name)
             repo (state/get-current-repo)]
-      (if (or (not exist?) (true? overwrite))
-        (do (when-let [old-target (and exist? (db-model/get-template-by-name template-name))]
+      (if (or (not block) (true? overwrite))
+        (do (when-let [old-target block]
               (property-handler/remove-block-property! repo (:block/uuid old-target) :template))
             (property-handler/set-block-property! repo target-uuid :template template-name))
         (throw (js/Error. "Template already exists!"))))))
 
 (defn ^:export remove_template
   [name]
-  (when-let [target (db-model/get-template-by-name name)]
-    (property-handler/remove-block-property!
-     (state/get-current-repo)
-     (:block/uuid target) :template)))
+  (p/let [block (when name (db-async/<get-template-by-name name))]
+    (when block
+      (property-handler/remove-block-property!
+       (state/get-current-repo)
+       (:block/uuid block) :template))))
 
 ;; search
 (defn ^:export search

src/test/frontend/db/model_test.cljs

@@ -51,8 +51,8 @@
     (are [x y] (= x y)
          4 (count a-aliases)
          4 (count b-aliases)
-         4 (count b-ref-blocks)
-         4 (count a-ref-blocks)
+         2 (count b-ref-blocks)
+         2 (count a-ref-blocks)
          #{"ab" "ac" "ad"} (set alias-names))))
 
 (deftest test-page-alias-set
@@ -66,9 +66,9 @@
         alias-names (model/get-page-alias-names test-helper/test-db "aa")
         a-ref-blocks (model/get-page-referenced-blocks "aa")]
     (are [x y] (= x y)
-         3 (count a-aliases)
-         3 (count a-ref-blocks)
-         #{"ab" "ac"} (set alias-names))))
+      3 (count a-aliases)
+      2 (count a-ref-blocks)
+      #{"ab" "ac"} (set alias-names))))
 
 (deftest get-pages-that-mentioned-page-with-show-journal
   (load-test-files [{:file/path "journals/2020_08_15.md"