Performance improvement: incremental queries (#4002)

Incremental queries

src/main/frontend/components/block.cljs

@@ -576,7 +576,7 @@
 
 (rum/defc block-embed < rum/reactive db-mixins/query
   [config id]
-  (let [blocks (db/get-block-and-children (state/get-current-repo) id)]
+  (let [blocks (db/sub-block-and-children (state/get-current-repo) id)]
     [:div.color-level.embed-block.bg-base-2
      {:style {:z-index 2}
       :on-double-click #(edit-parent-block % config)
@@ -636,6 +636,7 @@
 (declare block-parents)
 
 (rum/defc block-reference < rum/reactive
+  db-mixins/query
   [config id label]
   (when (and
          (not (string/blank? id))
@@ -685,14 +686,15 @@
                       :else (route-handler/redirect-to-page! id))))))}
 
            (if (and (not (util/mobile?)) (not (:preview? config)) (nil? block-type))
-             (ui/tippy {:html        [:div.tippy-wrapper.overflow-y-auto.p-4
-                                      {:style {:width      735
-                                               :text-align "left"
-                                               :max-height 600}}
-                                      [(block-parents config repo block-id {:indent? true})
-                                       (blocks-container
-                                        (db/get-block-and-children repo block-id)
-                                        (assoc config :id (str id) :preview? true))]]
+             (ui/tippy {:html        (fn []
+                                       [:div.tippy-wrapper.overflow-y-auto.p-4
+                                        {:style {:width      735
+                                                 :text-align "left"
+                                                 :max-height 600}}
+                                        [(block-parents config repo block-id {:indent? true})
+                                         (blocks-container
+                                          (db/get-block-and-children repo block-id)
+                                          (assoc config :id (str id) :preview? true))]])
                         :interactive true
                         :delay       [1000, 100]} inner)
              inner)])
@@ -1924,9 +1926,9 @@
         [:p.warning.text-sm "Full content is not displayed, Logseq doesn't support multiple unordered lists or headings in a block."]
         nil)]]))
 
-(rum/defc block-refs-count < rum/reactive
+(rum/defc block-refs-count < rum/reactive db-mixins/query
   [block]
-  (let [block-refs-count (model/get-block-refs-count (:block/uuid block))]
+  (let [block-refs-count (model/get-block-refs-count (:db/id block))]
     (when (and block-refs-count (> block-refs-count 0))
       [:div
        [:a.open-block-ref-link.bg-base-2.text-sm.ml-2.fade-link
@@ -2898,14 +2900,16 @@
                                    flat-blocks
                                    @*last-idx
                                    blocks->vec-tree)
-        bottom-reached (fn []
-                         (reset! *last-idx idx))
+        bottom-reached (fn [] (reset! *last-idx idx))
         has-more? (>= (count flat-blocks) (inc idx))]
     [:div#lazy-blocks
      (ui/infinite-list
       "main-content-container"
       (block-list config segment)
       {:on-load bottom-reached
+       :bottom-reached (fn []
+                         (when-let [node (gdom/getElement "lazy-blocks")]
+                           (ui/bottom-reached? node 1000)))
        :has-more has-more?
        :more (if (or (:preview? config) (:sidebar? config))
                "More"

src/main/frontend/components/journal.cljs

@@ -92,9 +92,9 @@
   [latest-journals]
   [:div#journals
    (ui/infinite-list "main-content-container"
-                     (for [[journal-name format] latest-journals]
-                       [:div.journal-item.content {:key journal-name}
-                        (journal-cp [journal-name format])])
+                     (for [{:block/keys [name format]} latest-journals]
+                       [:div.journal-item.content {:key name}
+                        (journal-cp [name format])])
                      {:has-more (page-handler/has-more-journals?)
                       :more-class "text-4xl"
                       :on-top-reached page-handler/create-today-journal!

src/main/frontend/components/page.cljs

@@ -45,10 +45,12 @@
   (when page-name
     (if block-id
       (when-let [root-block (db/pull [:block/uuid block-id])]
-        (let [blocks (-> (db/get-block-and-children repo block-id)
+        (let [blocks (-> (db/sub-block-and-children repo block-id)
                          (model/sort-blocks root-block {}))]
           (cons root-block blocks)))
-      (db/get-page-blocks repo page-name))))
+      (when-let [page (db/pull [:block/name (util/safe-page-name-sanity-lc page-name)])]
+        (-> (db/get-page-blocks repo page-name)
+            (model/sort-blocks page {}))))))
 
 (defn- open-first-block!
   [state]

src/main/frontend/components/sidebar.cljs

@@ -138,8 +138,7 @@
               (let [icon (get-page-icon entity)]
                 (favorite-item t name icon)))))]))))
 
-(rum/defc recent-pages
-  < rum/reactive db-mixins/query
+(rum/defc recent-pages < rum/reactive db-mixins/query
   [t]
   (nav-content-item
    [:a.flex.items-center.text-sm.font-medium.rounded-md
@@ -200,7 +199,7 @@
     [:span.flex-1 title]]])
 
 (rum/defc sidebar-nav
-  [_route-match close-modal-fn]
+  [_route-match close-modal-fn left-sidebar-open?]
   (let [default-home (get-default-home-if-valid)]
 
     [:div.left-sidebar-inner.flex-1.flex.flex-col.min-h-0
@@ -213,7 +212,6 @@
        (repo/repos-dropdown)
 
        [:div.nav-header
-
         (if (:page default-home)
           (sidebar-item
            {:class            "home-nav"
@@ -243,7 +241,7 @@
 
       (favorites t)
 
-      (recent-pages t)
+      (when left-sidebar-open? (recent-pages t))
 
       [:nav.px-2 {:aria-label "Sidebar"
                   :class      "new-page"}
@@ -263,7 +261,7 @@
      {:class (util/classnames [{:is-open left-sidebar-open?}])}
 
      ;; sidebar contents
-     (sidebar-nav route-match close-fn)
+     (sidebar-nav route-match close-fn left-sidebar-open?)
      [:span.shade-mask {:on-click close-fn}]]))
 
 (rum/defc main <

src/main/frontend/db.cljs

@@ -7,7 +7,7 @@
             [frontend.db.model]
             [frontend.db.query-custom]
             [frontend.db.query-react]
-            [frontend.db.react]
+            [frontend.db.react :as react]
             [frontend.db.utils]
             [frontend.db.persist :as db-persist]
             [frontend.db.migrate :as db-migrate]
@@ -35,15 +35,15 @@
   entity pull pull-many transact! get-key-value]
 
  [frontend.db.model
-  block-and-children-transform blocks-count blocks-count-cache clean-export!  cloned? delete-blocks get-pre-block
+  blocks-count blocks-count-cache clean-export!  cloned? delete-blocks get-pre-block
   delete-file! delete-file-blocks! delete-page-blocks delete-file-pages! delete-file-tx delete-files delete-pages-by-files
   filter-only-public-pages-and-blocks get-all-block-contents get-all-tagged-pages
-  get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
+  get-all-templates get-block-and-children sub-block-and-children get-block-by-uuid get-block-children sort-by-left
   get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks
   get-block-children-ids get-block-immediate-children get-block-page
   get-blocks-contents get-custom-css
-  get-date-scheduled-or-deadlines get-db-type get-file
-  get-file-blocks get-file-contents get-file-last-modified-at get-file-no-sub get-file-page get-file-page-id file-exists?
+  get-date-scheduled-or-deadlines get-db-type
+  get-file-blocks get-file-contents get-file-last-modified-at get-file get-file-page get-file-page-id file-exists?
   get-file-pages get-files get-files-blocks get-files-full get-journals-length
   get-latest-journals get-matched-blocks get-page get-page-alias get-page-alias-names get-page-blocks get-page-linked-refs-refed-pages
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
@@ -54,9 +54,9 @@
   set-file-content! has-children? get-namespace-pages get-all-namespace-relation get-pages-by-name-partition]
 
  [frontend.db.react
-  get-current-marker get-current-page get-current-priority set-key-value
-  transact-react! remove-key! remove-q! remove-query-component! add-q! add-query-component! clear-query-state!
-  clear-query-state-without-refs-and-embeds! get-block-blocks-cache-atom get-page-blocks-cache-atom kv q
+  get-current-page set-key-value
+  remove-key! remove-q! remove-query-component! add-q! add-query-component! clear-query-state!
+  clear-query-state-without-refs-and-embeds! kv q
   query-state query-components query-entity-in-component remove-custom-query! set-new-result! sub-key-value refresh!]
 
  [frontend.db.query-custom
@@ -120,6 +120,9 @@
   [repo conn]
   (d/listen! conn :persistence
              (fn [tx-report]
+               ;; reactive components
+               (react/refresh! repo tx-report)
+
                (when-not (:new-graph? (:tx-meta tx-report)) ; skip initial txs
                  (if (util/electron?)
                    (when-not (:dbsync? (:tx-meta tx-report))

src/main/frontend/db/model.cljs

@@ -65,14 +65,14 @@
   (let [repo (state/get-current-repo)]
     (when (conn/get-conn repo)
       (->
-       (react/q repo [:blocks id] {}
-                '[:find (pull ?block [*])
-                  :in $ ?id
-                  :where
-                  [?block :block/uuid ?id]]
-                id)
+       (react/q repo [:frontend.db.react/block id] {}
+         '[:find [(pull ?block [*]) ...]
+           :in $ ?id
+           :where
+           [?block :block/uuid ?id]]
+         id)
        react
-       ffirst))))
+       first))))
 
 (defn get-tag-pages
   [repo tag-name]
@@ -214,8 +214,9 @@
   (when (and repo path last-modified-at)
     (when-let [conn (conn/get-conn repo false)]
       (d/transact! conn
-                   [{:file/path path
-                     :file/last-modified-at last-modified-at}]))))
+        [{:file/path path
+          :file/last-modified-at last-modified-at}]
+        {:skip-refresh? true}))))
 
 (defn get-file-last-modified-at
   [repo path]
@@ -230,23 +231,6 @@
     (when-let [conn (conn/get-conn repo false)]
       (d/entity (d/db conn) [:file/path path]))))
 
-(defn get-file
-  ([path]
-   (get-file (state/get-current-repo) path))
-  ([repo path]
-   (when (and repo path)
-     (->
-      (react/q repo [:file/content path]
-               {:use-cache? true}
-               '[:find ?content
-                 :in $ ?path
-                 :where
-                 [?file :file/path ?path]
-                 [?file :file/content ?content]]
-               path)
-      react
-      ffirst))))
-
 (defn get-file-contents
   [repo]
   (when-let [conn (conn/get-conn repo)]
@@ -271,19 +255,19 @@
       conn)
      (flatten))))
 
-(defn get-custom-css
-  []
-  (when-let [repo (state/get-current-repo)]
-    (get-file (config/get-file-path repo "logseq/custom.css"))))
-
-(defn get-file-no-sub
+(defn get-file
   ([path]
-   (get-file-no-sub (state/get-current-repo) path))
+   (get-file (state/get-current-repo) path))
   ([repo path]
    (when (and repo path)
      (when-let [conn (conn/get-conn repo)]
        (:file/content (d/entity conn [:file/path path]))))))
 
+(defn get-custom-css
+  []
+  (when-let [repo (state/get-current-repo)]
+    (get-file (config/get-file-path repo "logseq/custom.css"))))
+
 (defn get-block-by-uuid
   [id]
   (db-utils/entity [:block/uuid (if (uuid? id) id (uuid id))]))
@@ -449,36 +433,15 @@
               result)))))))
 
 (defn get-block-refs-count
-  [block-id]
+  [block-db-id]
   (when-let [repo-url (state/get-current-repo)]
-    (when block-id
+    (when block-db-id
       (some->
-      (react/q repo-url [:block/refs-count block-id]
-        {:query-fn (fn [_db]
-                     (count (:block/_refs (db-utils/entity repo-url [:block/uuid block-id]))))}
-        nil)
-      react))))
-
-;; TODO: native sort and limit support in DB
-(defn- get-limited-blocks
-  [db page block-eids limit]
-  (let [lefts (d/datoms db :avet :block/left)
-        lefts (zipmap (map :e lefts) lefts)
-        collapsed (d/datoms db :avet :block/collapsed?)
-        collapsed (zipmap (map :e collapsed) collapsed)
-        parents (d/datoms db :avet :block/parent)
-        parents (zipmap (map :e parents) parents)
-        blocks (map (fn [id]
-                      (let [collapsed? (:v (get collapsed id))]
-                        (cond->
-                          {:db/id id
-                           :block/left {:db/id (:v (get lefts id))}
-                           :block/parent {:db/id (:v (get parents id))}}
-                          collapsed?
-                          (assoc :block/collapsed? true))))
-                 block-eids)
-        blocks (sort-blocks blocks page limit)]
-    (map :db/id blocks)))
+       (react/q repo-url [:frontend.db.react/block-refs-count block-db-id]
+         {:query-fn (fn [_db]
+                      (count (:block/_refs (db-utils/entity repo-url block-db-id))))}
+         nil)
+       react))))
 
 ;; Use datoms index and provide limit support
 (defn get-page-blocks
@@ -486,7 +449,7 @@
    (get-page-blocks (state/get-current-repo) page nil))
   ([repo-url page]
    (get-page-blocks repo-url page nil))
-  ([repo-url page {:keys [use-cache? pull-keys limit]
+  ([repo-url page {:keys [use-cache? pull-keys]
                    :or {use-cache? true
                         pull-keys '[*]}}]
    (when page
@@ -502,19 +465,12 @@
                           :block/journal-day (:block/journal-day page-entity)}]
        (when page-id
          (some->
-          (react/q repo-url [:page/blocks page-id]
+          (react/q repo-url [:frontend.db.react/page-blocks page-id]
             {:use-cache? use-cache?
              :query-fn (fn [db]
                          (let [datoms (d/datoms db :avet :block/page page-id)
                                block-eids (mapv :e datoms)
-                               ;; TODO: needs benchmark
-                               long-page? (> (count datoms) 1000)
-                               block-eids (if long-page?
-                                            (get-limited-blocks db page-entity block-eids limit)
-                                            block-eids)
-                               blocks (db-utils/pull-many repo-url pull-keys block-eids)
-                               blocks (if long-page? blocks
-                                          (sort-blocks blocks page-entity nil))]
+                               blocks (db-utils/pull-many repo-url pull-keys block-eids)]
                            (map (fn [b] (assoc b :block/page bare-page-map)) blocks)))}
             nil)
           react))))))
@@ -542,10 +498,12 @@
 
 (defn page-empty?
   [repo page-id]
-  (let [page-id (if (integer? page-id)
-                  page-id
-                  [:block/name (util/safe-page-name-sanity-lc page-id)])]
-    (empty? (:block/_parent (db-utils/entity repo page-id)))))
+  (when-let [db (conn/get-conn repo)]
+    (let [page-id (if (string? page-id)
+                    [:block/name (util/safe-page-name-sanity-lc page-id)]
+                    page-id)
+          page (d/entity db page-id)]
+      (nil? (:block/_left page)))))
 
 (defn page-empty-or-dummy?
   [repo page-id]
@@ -624,9 +582,6 @@
                               '[:db/id :block/name :block/original-name]
                               ids))))))
 
-(defn block-and-children-transform
-  [result _repo-url _block-uuid]
-  (db-utils/seq-flatten result))
 
 (defn get-block-children-ids
   [repo block-uuid]
@@ -674,12 +629,24 @@
     [blocks-tree]))
 
 (defn get-block-and-children
+  [repo block-uuid]
+  (some-> (d/q
+            '[:find [(pull ?block ?block-attrs) ...]
+              :in $ ?id ?block-attrs
+              :where
+              [?block :block/uuid ?id]]
+            (conn/get-conn repo)
+            block-uuid
+            block-attrs)
+          first
+          flatten-tree))
+
+(defn sub-block-and-children
   ([repo block-uuid]
-   (get-block-and-children repo block-uuid true))
+   (sub-block-and-children repo block-uuid true))
   ([repo block-uuid use-cache?]
-   (some-> (react/q repo [:block/block block-uuid]
-             {:use-cache? use-cache?
-              :transform-fn #(block-and-children-transform % repo block-uuid)}
+   (some-> (react/q repo [:frontend.db.react/block-and-children block-uuid]
+             {:use-cache? use-cache?}
              '[:find [(pull ?block ?block-attrs) ...]
                :in $ ?id ?block-attrs
                :where
@@ -795,27 +762,21 @@
    (when (conn/get-conn repo-url)
      (let [date (js/Date.)
            _ (.setDate date (- (.getDate date) (dec n)))
-           today (db-utils/date->int (js/Date.))
-           pages (->>
-                  (react/q repo-url [:journals] {:use-cache? false}
-                           '[:find ?page-name ?journal-day
-                             :in $ ?today
-                             :where
-                             [?page :block/name ?page-name]
-                             [?page :block/journal? true]
-                             [?page :block/journal-day ?journal-day]
-                             [(<= ?journal-day ?today)]]
-                           today)
-                  (react)
-                  (sort-by last)
-                  (reverse)
-                  (map first)
-                  (take n))]
-       (mapv
-        (fn [page]
-          [page
-           (get-page-format page)])
-        pages)))))
+           today (db-utils/date->int (js/Date.))]
+       (->>
+        (react/q repo-url [:frontend.db.react/journals] {:use-cache? false}
+          '[:find [(pull ?page [*]) ...]
+            :in $ ?today
+            :where
+            [?page :block/name ?page-name]
+            [?page :block/journal? true]
+            [?page :block/journal-day ?journal-day]
+            [(<= ?journal-day ?today)]]
+          today)
+        (react)
+        (sort-by :block/journal-day)
+        (reverse)
+        (take n))))))
 
 ;; get pages that this page referenced
 (defn get-page-referenced-pages
@@ -884,7 +845,7 @@
   (when (conn/get-conn repo)
     (let [page-id (:db/id (db-utils/entity [:block/name (util/safe-page-name-sanity-lc page)]))
           pages (page-alias-set repo page)
-          mentioned-pages (->> (react/q repo [:page/mentioned-pages page-id] {:use-cache? false}
+          mentioned-pages (->> (react/q repo [:frontend.db.react/page<-pages page-id] {:use-cache? false}
                                         '[:find ?mentioned-page-name
                                           :in $ ?pages ?page-name
                                           :where
@@ -936,22 +897,26 @@
                                           [(find-blocks ?block ?ref-page ?pages ?alias ?aliases)
                                            [?block :block/refs ?ref-page]
                                            [(contains? ?pages ?ref-page)]]]]
-                              (react/q repo [:block/refed-blocks page-id] {}
-                                '[:find [(pull ?block ?block-attrs) ...]
-                                  :in $ % ?pages ?aliases ?block-attrs
-                                  :where
-                                  (find-blocks ?block ?ref-page ?pages ?alias ?aliases)]
-                                rules
-                                pages
-                                aliases
-                                block-attrs))
-                            (react/q repo [:block/refed-blocks page-id] {:use-cache? false}
-                              '[:find [(pull ?ref-block ?block-attrs) ...]
-                                :in $ ?page ?block-attrs
-                                :where
-                                [?ref-block :block/refs ?page]]
-                              page-id
-                              block-attrs))
+                              (react/q repo
+                                       [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
+                                       {}
+                                       '[:find [(pull ?block ?block-attrs) ...]
+                                         :in $ % ?pages ?aliases ?block-attrs
+                                         :where
+                                         (find-blocks ?block ?ref-page ?pages ?alias ?aliases)]
+                                       rules
+                                       pages
+                                       aliases
+                                       block-attrs))
+                            (react/q repo
+                                     [:frontend.db.react/page<-blocks-or-block<-blocks page-id]
+                                     {:use-cache? false}
+                                     '[:find [(pull ?ref-block ?block-attrs) ...]
+                                       :in $ ?page ?block-attrs
+                                       :where
+                                       [?ref-block :block/refs ?page]]
+                                     page-id
+                                     block-attrs))
              result (->> query-result
                          react
                          (sort-by-left-recursive)
@@ -1047,7 +1012,7 @@
                              (map pattern))
             filter-fn   (fn [datom]
                           (some (fn [p] (re-find p (:v datom))) patterns))]
-        (->> (react/q repo [:block/unlinked-refs page-id]
+        (->> (react/q repo [:frontend.db.react/page-unlinked-refs page-id]
                       {:query-fn (fn [db]
                                    (let [ids
                                          (->> (d/datoms db :aevt :block/content)
@@ -1067,16 +1032,17 @@
   (when-let [repo (state/get-current-repo)]
     (when (conn/get-conn repo)
       (let [block (db-utils/entity [:block/uuid block-uuid])]
-        (->> (react/q repo [:block/refed-blocks (:db/id block)]
-               {}
-               '[:find [(pull ?ref-block ?block-attrs) ...]
-                 :in $ ?block-uuid ?block-attrs
-                :where
-                [?block :block/uuid ?block-uuid]
-                [?ref-block :block/refs ?block]]
-               block-uuid
-               block-attrs)
-            react
+        (->> (react/q repo [:frontend.db.react/page<-blocks-or-block<-blocks
+                            (:db/id block)]
+                      {:use-cache? false}
+                      '[:find [(pull ?ref-block ?block-attrs) ...]
+                        :in $ ?block-uuid ?block-attrs
+                        :where
+                        [?block :block/uuid ?block-uuid]
+                        [?ref-block :block/refs ?block]]
+                      block-uuid
+                      block-attrs)
+             react
             (sort-by-left-recursive)
             db-utils/group-by-page)))))
 
@@ -1085,14 +1051,15 @@
   (when-let [repo (state/get-current-repo)]
     (when-let [conn (conn/get-conn repo)]
       (let [block (db-utils/entity [:block/uuid block-uuid])]
-        (->> (react/q repo [:ref-ids (:db/id block)] {}
-               '[:find ?ref-block
-                 :in $ ?block-uuid ?block-attrs
-                 :where
-                 [?block :block/uuid ?block-uuid]
-                 [?ref-block :block/refs ?block]]
-               block-uuid
-               block-attrs)
+        (->> (react/q repo [:frontend.db.react/block<-block-ids
+                            (:db/id block)] {}
+                      '[:find ?ref-block
+                        :in $ ?block-uuid ?block-attrs
+                        :where
+                        [?block :block/uuid ?block-uuid]
+                        [?ref-block :block/refs ?block]]
+                      block-uuid
+                      block-attrs)
              react)))))
 
 (defn get-referenced-blocks-ids
@@ -1361,10 +1328,7 @@
   (when (and repo path)
     (let [tx-data {:file/path path
                    :file/content content}]
-      (react/transact-react!
-       repo
-       [tx-data]
-       {:key [:file/content path]}))))
+      (db-utils/transact! repo [tx-data] {:skip-refresh? true}))))
 
 (defn get-pre-block
   [repo page-id]

src/main/frontend/db/react.cljs

@@ -4,15 +4,62 @@
   It'll be great if we can find an automatically resolving and performant
   solution.
   "
-  (:require [clojure.string :as string]
-            [datascript.core :as d]
-            [frontend.config :as config]
+  (:require [datascript.core :as d]
             [frontend.date :as date]
             [frontend.db.conn :as conn]
             [frontend.db.utils :as db-utils]
             [frontend.state :as state]
             [frontend.util :as util :refer [react]]
-            [frontend.util.marker :as marker]))
+            [frontend.db-schema :as db-schema]
+            [cljs.spec.alpha :as s]))
+
+;;; keywords specs for reactive query, used by `react/q` calls
+;; ::block
+;; pull-block react-query
+(s/def ::block (s/tuple #(= ::block %) uuid?))
+;; ::block-refs-count
+;; (count (:block/refs block)) react-query
+(s/def ::block-refs-count (s/tuple #(= ::block-refs-count %) int?))
+;; ::page-blocks
+;; get page-blocks react-query
+(s/def ::page-blocks (s/tuple #(= ::page-blocks %) int?))
+;; ::block-and-children
+;; get block&children react-query
+(s/def ::block-and-children (s/tuple #(= ::block-and-children %) uuid?))
+;; ::journals
+;; get journal-list react-query
+(s/def ::journals (s/tuple #(= ::journals %)))
+;; ::page->pages
+;; get PAGES referenced by PAGE
+(s/def ::page->pages (s/tuple #(= ::page->pages %) int?))
+;; ::page<-pages
+;; get PAGES referencing PAGE
+(s/def ::page<-pages (s/tuple #(= ::page<-pages %) int?))
+;; ::page<-blocks-or-block<-blocks
+;; get BLOCKS referencing PAGE or BLOCK
+(s/def ::page<-blocks-or-block<-blocks
+  (s/tuple #(= ::page<-blocks-or-block<-blocks %) int?))
+;; FIXME: this react-query has performance issues
+(s/def ::page-unlinked-refs (s/tuple #(= ::page-unlinked-refs %) int?))
+;; ::block<-block-ids
+;; get BLOCK-IDS referencing BLOCK
+(s/def ::block<-block-ids (s/tuple #(= ::block<-block-ids %) int?))
+;; custom react-query
+(s/def ::custom any?)
+
+(s/def ::react-query-keys (s/or :block ::block
+                                :block-refs-count ::block-refs-count
+                                :page-blocks ::page-blocks
+                                :block-and-children ::block-and-children
+                                :journals ::journals
+                                :page->pages ::page->pages
+                                :page<-pages ::page<-pages
+                                :page<-blocks-or-block<-blocks ::page<-blocks-or-block<-blocks
+                                :page-unlinked-refs ::page-unlinked-refs
+                                :block<-block-ids ::block<-block-ids
+                                :custom ::custom))
+
+(s/def ::affected-keys (s/coll-of ::react-query-keys))
 
 ;; Query atom of map of Key ([repo q inputs]) -> atom
 ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items?
@@ -29,8 +76,6 @@
   (when-let [result-atom (get-in @query-state [k :result])]
     (reset! result-atom new-result)))
 
-;; KV
-
 (defn kv
   [key value]
   {:db/id -1
@@ -54,24 +99,6 @@
                    (into {}))]
     (reset! query-state state)))
 
-(defn get-current-repo-refs-keys
-  [{:keys [data]}]
-  (when-let [current-repo (state/get-current-repo)]
-    (->>
-     (map (fn [[repo k id]]
-            (when (and (= repo current-repo)
-                       (contains? #{:block/refed-blocks :block/unlinked-refs} k))
-              (if (= k :block/refed-blocks)
-                (if (every? (fn [m]
-                              (when (map? m)
-                                (= id (:db/id (:block/page m))))) data)
-                  nil
-                  [k id])
-                [k id])))
-       (keys @query-state))
-     (remove nil?))))
-
-;; TODO: Add components which subscribed to a specific query
 (defn add-q!
   [k query inputs result-atom transform-fn query-fn inputs-fn]
   (swap! query-state assoc k {:query query
@@ -84,7 +111,8 @@
 
 (defn remove-q!
   [k]
-  (swap! query-state dissoc k))
+  (swap! query-state dissoc k)
+  (state/delete-reactive-query-db! k))
 
 (defn add-query-component!
   [key component]
@@ -105,14 +133,6 @@
         (keep identity)
         (into {}))))
 
-(defn get-page-blocks-cache-atom
-  [repo page-id]
-  (:result (get @query-state [repo :page/blocks page-id])))
-
-(defn get-block-blocks-cache-atom
-  [repo block-id]
-  (:result (get @query-state [repo :block/block block-id])))
-
 ;; TODO: rename :custom to :query/custom
 (defn remove-custom-query!
   [repo query]
@@ -135,10 +155,29 @@
          (set! (.-state result-atom) result)
          (add-q! k nil nil result-atom identity identity identity))))))
 
+(defn- new-db
+  [cached-result tx-data old-db k]
+  (when (and (coll? cached-result)
+             (map? (first cached-result)))
+    (try
+      (let [db (or old-db
+                   (let [cached-result (util/remove-nils cached-result)]
+                     (-> (d/empty-db db-schema/schema)
+                         (d/with cached-result)
+                         (:db-after))))]
+        (:db-after (d/with db tx-data)))
+      (catch js/Error e
+        (prn "New db: " {:k k
+                         :old-db old-db
+                         :cached-result cached-result})
+        (js/console.error e)
+        old-db))))
+
 (defn q
   [repo k {:keys [use-cache? transform-fn query-fn inputs-fn disable-reactive?]
            :or {use-cache? true
                 transform-fn identity}} query & inputs]
+  {:pre [(s/valid? ::react-query-keys k)]}
   (let [kv? (and (vector? k) (= :kv (first k)))
         k (vec (cons repo k))]
     (when-let [conn (conn/get-conn repo)]
@@ -167,10 +206,12 @@
                 result-atom (or result-atom (atom nil))]
             ;; Don't notify watches now
             (set! (.-state result-atom) result)
-            (if-not disable-reactive?
-              (add-q! k query inputs result-atom transform-fn query-fn inputs-fn)
-              result-atom)))))))
-
+            (if disable-reactive?
+              result-atom
+              (do
+                (let [db' (new-db result nil nil k)]
+                  (state/set-reactive-query-db! k db'))
+                (add-q! k query inputs result-atom transform-fn query-fn inputs-fn)))))))))
 
 
 ;; TODO: Extract several parts to handlers
@@ -192,159 +233,112 @@
       (let [page-name (util/page-name-sanity-lc page)]
         (db-utils/entity [:block/name page-name])))))
 
-(defn get-current-priority
-  []
-  (let [match (:route-match @state/state)
-        route-name (get-in match [:data :name])]
-    (when (= route-name :page)
-      (when-let [page-name (get-in match [:path-params :name])]
-        (and (contains? #{"a" "b" "c"} (string/lower-case page-name))
-             (string/upper-case page-name))))))
+(defn get-affected-queries-keys
+  "Get affected queries through transaction datoms."
+  [{:keys [tx-data]}]
+  {:post [(s/valid? ::affected-keys %)]}
+
+  (let [blocks (->> (filter (fn [datom] (contains? #{:block/left :block/parent :block/page} (:a datom))) tx-data)
+                    (map :v)
+                    (distinct))
+        refs (->> (filter (fn [datom] (= :block/refs (:a datom))) tx-data)
+                  (map :v)
+                  (distinct))
+        other-blocks (->> (filter (fn [datom] (= "block" (namespace (:a datom)))) tx-data)
+                          (map :e))
+        blocks (-> (concat blocks other-blocks) distinct)
+        affected-keys (concat
+                       (mapcat
+                        (fn [block-id]
+                          (let [block-id (if (and (string? block-id) (util/uuid-string? block-id))
+                                           [:block/uuid block-id]
+                                           block-id)]
+                            (when-let [block (db-utils/entity block-id)]
+                              (let [page-id (or
+                                             (when (:block/name block) (:db/id block))
+                                             (:db/id (:block/page block)))
+                                    blocks [[::block (:block/uuid block)]]
+                                    others (when page-id
+                                             [[::page-blocks page-id]
+                                              [::page->pages page-id]])]
+                                (concat blocks others)))))
+                        blocks)
+
+                       (when-let [current-page-id (:db/id (get-current-page))]
+                         [[::page->pages current-page-id]
+                          [::page<-pages current-page-id]])
+
+                       (map (fn [ref]
+                              (let [entity (db-utils/entity ref)]
+                                (if (:block/name entity) ; page
+                                  [::page-blocks ref]
+                                  [::block-refs-count ref])))
+                            refs))
+        others (some->>
+                (filter (fn [ks]
+                          (contains? #{::block-and-children
+                                       ::page<-blocks-or-block<-blocks}
+                                     (second ks)))
+                        (keys @query-state))
+                (map (fn [v] (vec (rest v)))))]
+    (->>
+     (util/concat-without-nil
+      affected-keys
+      others)
+     set)))
 
-(defn get-current-marker
-  []
-  (let [match (:route-match @state/state)
-        route-name (get-in match [:data :name])]
-    (when (= route-name :page)
-      (when-let [page-name (get-in match [:path-params :name])]
-        (and (marker/marker? page-name)
-             (string/upper-case page-name))))))
-
-(defn get-related-keys
-  [{:keys [key data]}]
-  (cond
-    (coll? key)
-    [key]
-
-    :else
-    (case key
-      (:block/change :block/insert)
-      (when-let [blocks (seq data)]
-        (let [pre-block? (:block/pre-block? (first blocks))
-              current-priority (get-current-priority)
-              current-marker (get-current-marker)
-              current-page-id (:db/id (get-current-page))
-              related-keys (->>
-                            (util/concat-without-nil
-                             (mapcat
-                              (fn [block]
-                                (when-let [page-id (or (:db/id (:block/page block))
-                                                       (and (int? (:block/page block))
-                                                            (:block/page block)))]
-                                  [[:blocks (:block/uuid block)]
-                                   [:page/blocks page-id]
-                                   [:page/ref-pages page-id]]))
-                              blocks)
-
-                             (when pre-block?
-                               [[:contents]
-                                [:page/published]])
-
-                             ;; affected priority
-                             (when current-priority
-                               [[:priority/blocks current-priority]])
-
-                             (when current-marker
-                               [[:marker/blocks current-marker]])
-
-                             (when current-page-id
-                               [[:page/ref-pages current-page-id]
-                                [:page/mentioned-pages current-page-id]])
-
-                             (apply concat
-                               (for [{:block/keys [refs]} blocks]
-                                 (map (fn [ref]
-                                        (cond
-                                          (and (map? ref) (:block/name ref))
-                                          [:page/blocks (:db/id (db-utils/entity [:block/name (:block/name ref)]))]
-
-                                          (and (vector? ref) (= (first ref) :block/uuid))
-                                          [:block/refs-count (second ref)]
-
-                                          :else
-                                          nil))
-                                   refs))))
-                            (distinct))
-              refed-pages (map
-                           (fn [[k page-id]]
-                             (when (= k :block/refed-blocks)
-                               [:page/ref-pages page-id]))
-                            related-keys)
-              all-refed-blocks (get-current-repo-refs-keys {:key key
-                                                            :data data})
-              custom-queries (some->>
-                              (filter (fn [v]
-                                        (and (= (first v) (state/get-current-repo))
-                                             (= (second v) :custom)))
-                                      (keys @query-state))
-                              (map (fn [v]
-                                     (vec (drop 1 v)))))
-              block-blocks (some->>
-                            (filter (fn [v]
-                                      (and (= (first v) (state/get-current-repo))
-                                           (= (second v) :block/block)))
-                                    (keys @query-state))
-                            (map (fn [v]
-                                   (vec (drop 1 v)))))]
-          (->>
-           (util/concat-without-nil
-            related-keys
-            refed-pages
-            all-refed-blocks
-            custom-queries
-            block-blocks)
-           distinct)))
-      [[key]])))
-
-;; TODO: incremental or delayed queries (e.g. only run custom queries when idle)
 (defn refresh!
-  [repo-url handler-opts]
-  (let [related-keys (get-related-keys handler-opts)
-        db (conn/get-conn repo-url)]
-    (doseq [related-key related-keys]
-      (let [related-key (vec (cons repo-url related-key))]
-        (when-let [cache (get @query-state related-key)]
-          (let [{:keys [query inputs transform-fn query-fn inputs-fn]} cache]
+  "Re-compute corresponding queries (from tx) and refresh the related react components."
+  [repo-url {:keys [tx-data tx-meta] :as tx}]
+  (when (and repo-url
+             (seq tx-data)
+             (not (:skip-refresh? tx-meta)))
+    (let [db (conn/get-conn repo-url)
+          affected-keys (get-affected-queries-keys tx)]
+      (doseq [[k cache] @query-state]
+        (when (and
+               (= (first k) repo-url)
+               (or (get affected-keys (vec (rest k)))
+                   (= :custom (second k))))
+          (let [{:keys [query inputs transform-fn query-fn inputs-fn result]} cache]
             (when (or query query-fn)
-              (let [new-result (->
-                                (cond
-                                  query-fn
-                                  (let [result (query-fn db)]
-                                    (if (coll? result)
-                                      (doall result)
-                                      result))
-
-                                  inputs-fn
-                                  (let [inputs (inputs-fn)]
-                                    (apply d/q query db inputs))
-
-                                  (keyword? query)
-                                  (db-utils/get-key-value repo-url query)
-
-                                  (seq inputs)
-                                  (apply d/q query db inputs)
-
-                                  :else
-                                  (d/q query db))
-                                transform-fn)]
-                (set-new-result! related-key new-result)))))))))
-
-(defn transact-react!
-  [repo-url tx-data handler-opts]
-  (when-not config/publishing?
-    (let [repo-url (or repo-url (state/get-current-repo))
-          tx-data (->> (util/remove-nils tx-data)
-                       (remove nil?))
-          get-conn (fn [] (conn/get-conn repo-url false))]
-      (when (and (seq tx-data) (get-conn))
-        (d/transact! (get-conn) (vec tx-data))
-        (refresh! repo-url handler-opts)))))
+              (try
+                (let [db' (when (and (vector? k) (not= (second k) :kv))
+                            (let [query-db (state/get-reactive-query-db k)
+                                  result (new-db @result tx-data query-db k)]
+                              (state/set-reactive-query-db! k result)
+                              result))
+                      db (or db' db)
+                      new-result (->
+                                  (cond
+                                    query-fn
+                                    (let [result (query-fn db)]
+                                      (if (coll? result)
+                                        (doall result)
+                                        result))
+
+                                    inputs-fn
+                                    (let [inputs (inputs-fn)]
+                                      (apply d/q query db inputs))
+
+                                    (keyword? query)
+                                    (db-utils/get-key-value repo-url query)
+
+                                    (seq inputs)
+                                    (apply d/q query db inputs)
+
+                                    :else
+                                    (d/q query db))
+                                  transform-fn)]
+                  (when-not (= new-result result)
+                    (set-new-result! k new-result)))
+                (catch js/Error e
+                  (js/console.error e))))))))))
 
 (defn set-key-value
   [repo-url key value]
   (if value
-    (transact-react! repo-url [(kv key value)]
-                     {:key [:kv key]})
+    (db-utils/transact! repo-url [(kv key value)])
     (remove-key! repo-url key)))
 
 (defn sub-key-value

src/main/frontend/extensions/code.cljs

@@ -172,7 +172,7 @@
 
         (:file-path config)
         (let [path (:file-path config)
-              content (db/get-file-no-sub path)
+              content (db/get-file path)
               [_ id _ _ _] (:rum/args state)
               value (some-> (gdom/getElement id)
                             (gobj/get "value"))]

src/main/frontend/extensions/srs.cljs

@@ -7,6 +7,7 @@
             [frontend.util.drawer :as drawer]
             [frontend.util.persist-var :as persist-var]
             [frontend.db :as db]
+            [frontend.db-mixins :as db-mixins]
             [frontend.state :as state]
             [frontend.handler.editor :as editor-handler]
             [frontend.components.block :as component-block]
@@ -400,6 +401,7 @@
 
 (rum/defcs view
   < rum/reactive
+  db-mixins/query
   (rum/local 1 ::phase)
   (rum/local {} ::review-records)
   {:will-mount (fn [state]

src/main/frontend/handler/dnd.cljs

@@ -31,8 +31,7 @@
   [^js event current-block target-block move-to]
   (let [top? (= move-to :top)
         nested? (= move-to :nested)
-        alt-key? (and event (.-altKey event))
-        repo (state/get-current-repo)]
+        alt-key? (and event (.-altKey event))]
     (cond
       (not= (:block/format current-block) (:block/format target-block))
       (state/pub-event! [:notification/show
@@ -49,9 +48,7 @@
          (util/format "((%s))" (str (:block/uuid current-block)))
          {:block-uuid (:block/uuid target-block)
           :sibling? (not nested?)
-          :before? top?})
-        (db/refresh! repo {:key :block/change
-                           :data [current-block target-block]}))
+          :before? top?}))
 
       (and (every? map? [current-block target-block])
            (moveable? current-block target-block))
@@ -71,9 +68,7 @@
           (outliner-core/move-subtree current-node target-node false)
 
           :else
-          (outliner-core/move-subtree current-node target-node true))
-        (db/refresh! repo {:key :block/change
-                           :data [(:data current-node) (:data target-node)]}))
+          (outliner-core/move-subtree current-node target-node true)))
 
       :else
       nil)))

src/main/frontend/handler/editor.cljs

@@ -401,8 +401,7 @@
         (merge (if level {:block/level level} {})))))
 
 (defn- save-block-inner!
-  [repo block value {:keys [refresh?]
-                     :or {refresh? true}}]
+  [repo block value {}]
   (let [block (assoc block :block/content value)
         block (apply dissoc block db-schema/retract-attributes)]
     (profile
@@ -410,10 +409,6 @@
      (let [block (wrap-parse-block block)]
        (-> (outliner-core/block block)
            (outliner-core/save-node))
-       (when refresh?
-         (let [opts {:key :block/change
-                     :data [block]}]
-           (db/refresh! repo opts)))
 
        ;; sanitized page name changed
        (when-let [title (get-in block [:block/properties :title])]
@@ -461,7 +456,7 @@
       [fst-block-text snd-block-text])))
 
 (defn outliner-insert-block!
-  [config current-block new-block sibling?]
+  [config current-block new-block {:keys [sibling? txs-state]}]
   (let [ref-top-block? (and (:ref? config)
                             (not (:ref-child? config)))
         skip-save-current-block? (:skip-save-current-block? config)
@@ -480,9 +475,10 @@
                    true
 
                    :else
-                   (not has-children?))]
+                   (not has-children?))
+        txs-state' (or txs-state (ds/new-outliner-txs-state))]
     (ds/auto-transact!
-     [txs-state (ds/new-outliner-txs-state)]
+     [txs-state txs-state']
      {:outliner-op :save-and-insert-node
       :skip-transact? false}
      (let [*blocks (atom [current-node])]
@@ -505,52 +501,12 @@
                        current-page))]
     (= uuid (and block-id (medley/uuid block-id)))))
 
-;; FIXME: painful
-(defn update-cache-for-block-insert!
-  "Currently, this only affects current editor container to improve the performance."
-  [repo config {:block/keys [page uuid] :as _block} blocks]
-  (let [blocks (map :data blocks)
-        [first-block last-block right-block] blocks
-        child? (= (first (:block/parent last-block))
-                  (:block/uuid first-block))
-        blocks-container-id (when-let [id (:id config)]
-                              (and (util/uuid-string? id) (medley/uuid id)))
-        new-last-block (let [first-block-id {:db/id (:db/id first-block)}]
-                         (assoc last-block
-                                :block/left first-block-id
-                                :block/parent (if child?
-                                                first-block-id
-                                                ;; sibling
-                                                (:block/parent first-block))))
-        blocks [first-block new-last-block]
-        blocks-atom (if blocks-container-id
-                      (db/get-block-blocks-cache-atom repo blocks-container-id)
-                      (db/get-page-blocks-cache-atom repo (:db/id page)))
-        [before-part after-part] (and blocks-atom
-                                      (split-with
-                                       #(not= uuid (:block/uuid %))
-                                       @blocks-atom))
-        after-part (rest after-part)
-        blocks (concat before-part blocks after-part)
-        blocks (if right-block
-                 (map (fn [block]
-                        (if (= (:block/uuid right-block) (:block/uuid block))
-                          (assoc block :block/left (:block/left right-block))
-                          block)) blocks)
-                 blocks)]
-    (when blocks-atom
-      (reset! blocks-atom blocks))))
-
 (defn insert-new-block-before-block-aux!
-  [config
-   {:block/keys [repo]
-    :as block}
-   value
+  [config block value
    {:keys [ok-handler]
     :as _opts}]
   (let [input (gdom/getElement (state/get-edit-input-id))
         pos (cursor/pos input)
-        repo (or repo (state/get-current-repo))
         [fst-block-text snd-block-text] (compute-fst-snd-block-text value pos)
         current-block (assoc block :block/content snd-block-text)
         current-block (apply dissoc current-block db-schema/retract-attributes)
@@ -560,18 +516,19 @@
         prev-block (-> (merge (select-keys block [:block/parent :block/left :block/format
                                                   :block/page :block/journal?]) new-m)
                        (wrap-parse-block))
-        left-block (db/pull (:db/id (:block/left block)))
-        _ (outliner-core/save-node (outliner-core/block current-block))
-        sibling? (not= (:db/id left-block) (:db/id (:block/parent block)))]
+        left-block (db/pull (:db/id (:block/left block)))]
     (profile
      "outliner insert block"
-     (outliner-insert-block! config left-block prev-block sibling?))
-    (db/refresh! repo {:key :block/insert :data [prev-block left-block current-block]})
-    (profile "ok handler" (ok-handler prev-block))))
+     (let [txs-state (ds/new-outliner-txs-state)]
+       (outliner-core/save-node (outliner-core/block current-block) {:txs-state txs-state})
+       (let [sibling? (not= (:db/id left-block) (:db/id (:block/parent block)))]
+         (outliner-insert-block! config left-block prev-block {:sibling? sibling?
+                                                               :txs-state txs-state}))))
+    (ok-handler prev-block)))
 
 (defn insert-new-block-aux!
   [config
-   {:block/keys [uuid repo]
+   {:block/keys [uuid]
     :as block}
    value
    {:keys [ok-handler]
@@ -579,33 +536,19 @@
   (let [block-self? (block-self-alone-when-insert? config uuid)
         input (gdom/getElement (state/get-edit-input-id))
         pos (cursor/pos input)
-        repo (or repo (state/get-current-repo))
         [fst-block-text snd-block-text] (compute-fst-snd-block-text value pos)
         current-block (assoc block :block/content fst-block-text)
         current-block (apply dissoc current-block db-schema/retract-attributes)
         current-block (wrap-parse-block current-block)
-        zooming? (when-let [id (:id config)]
-                   (and (string? id) (util/uuid-string? id)))
         new-m {:block/uuid (db/new-block-id)
                :block/content snd-block-text}
         next-block (-> (merge (select-keys block [:block/parent :block/left :block/format
                                                   :block/page :block/journal?]) new-m)
                        (wrap-parse-block))
-        sibling? (when block-self? false)
-        {:keys [sibling? blocks]} (profile
-                                   "outliner insert block"
-                                   (outliner-insert-block! config current-block next-block sibling?))
-        refresh-fn (fn []
-                     (let [opts {:key :block/insert
-                                 :data [current-block next-block]}]
-                       (db/refresh! repo opts)))]
-    (if (or (:ref? config)
-            (not sibling?)
-            zooming?)
-      (refresh-fn)
-      (do
-        (profile "update cache " (update-cache-for-block-insert! repo config block blocks))
-        (state/add-tx! refresh-fn)))
+        sibling? (when block-self? false)]
+    (profile
+     "outliner insert block"
+     (outliner-insert-block! config current-block next-block {:sibling? sibling?}))
     ;; WORKAROUND: The block won't refresh itself even if the content is empty.
     (when block-self?
       (gobj/set input "value" ""))
@@ -728,8 +671,7 @@
                                    nil)]
 
           (when block-m
-            (outliner-insert-block! {:skip-save-current-block? true} block-m new-block sibling?)
-            (db/refresh! (state/get-current-repo) {:key :block/insert :data [block-m new-block]})
+            (outliner-insert-block! {:skip-save-current-block? true} block-m new-block {:sibling? sibling?})
             new-block))))))
 
 (defn insert-first-page-block-if-not-exists!
@@ -928,8 +870,7 @@
     (when block
       (->
        (outliner-core/block block)
-       (outliner-core/delete-node children?))
-      (db/refresh! repo {:key :block/change :data [block]}))))
+       (outliner-core/delete-node children?)))))
 
 (defn- move-to-prev-block
   [repo sibling-block format id value]
@@ -1021,11 +962,7 @@
             sibling-block (when block-parent (util/get-prev-block-non-collapsed-non-embed block-parent))]
         (if (= start-node end-node)
           (delete-block-aux! (first blocks) true)
-          (when (outliner-core/delete-nodes start-node end-node lookup-refs)
-            (let [opts {:key :block/change
-                        :data blocks}]
-              (db/refresh! repo opts)
-              (ui-handler/re-render-root!))))
+          (outliner-core/delete-nodes start-node end-node lookup-refs))
         (when sibling-block
           (move-to-prev-block repo sibling-block
                               (:block/format block)
@@ -1063,9 +1000,6 @@
     (let [block-id (ffirst col)
           block-id (if (string? block-id) (uuid block-id) block-id)
           input-pos (or (state/get-edit-pos) :max)]
-      (db/refresh! (state/get-current-repo)
-                   {:key :block/change
-                    :data [(db/pull [:block/uuid block-id])]})
       ;; update editing input content
       (when-let [editing-block (state/get-edit-block)]
         (when (= (:block/uuid editing-block) block-id)
@@ -1899,48 +1833,40 @@
   [up?]
   (fn [event]
     (util/stop event)
-    (when-let [repo (state/get-current-repo)]
-      (let [edit-block-id (:block/uuid (state/get-edit-block))
-            move-nodes (fn [blocks]
-                         (let [nodes (mapv outliner-core/block blocks)
-                               opts {:key :block/change
-                                     :data blocks}]
-                           (outliner-core/move-nodes nodes up?)
-                           (db/refresh! repo opts)
-                           (rehighlight-selected-nodes)
-                           (let [block-node (util/get-first-block-by-id (:block/uuid (first blocks)))]
-                             (.scrollIntoView block-node #js {:behavior "smooth" :block "nearest"}))))]
-        (if edit-block-id
-          (when-let [block (db/pull [:block/uuid edit-block-id])]
-            (let [blocks [block]]
-              (move-nodes blocks))
-            (when-let [input-id (state/get-edit-input-id)]
-              (when-let [input (gdom/getElement input-id)]
-                (.focus input))))
-          (let [blocks (-> (state/get-selection-blocks)
-                           reorder-selected-blocks)
-                blocks (filter #(= (:block/parent %) (:block/parent (first blocks))) blocks)]
-            (when (seq blocks)
-              (move-nodes blocks))))))))
+    (let [edit-block-id (:block/uuid (state/get-edit-block))
+          move-nodes (fn [blocks]
+                       (let [nodes (mapv outliner-core/block blocks)]
+                         (outliner-core/move-nodes nodes up?)
+                         (rehighlight-selected-nodes)
+                         (let [block-node (util/get-first-block-by-id (:block/uuid (first blocks)))]
+                           (.scrollIntoView block-node #js {:behavior "smooth" :block "nearest"}))))]
+      (if edit-block-id
+        (when-let [block (db/pull [:block/uuid edit-block-id])]
+          (let [blocks [block]]
+            (move-nodes blocks))
+          (when-let [input-id (state/get-edit-input-id)]
+            (when-let [input (gdom/getElement input-id)]
+              (.focus input))))
+        (let [blocks (-> (state/get-selection-blocks)
+                         reorder-selected-blocks)
+              blocks (filter #(= (:block/parent %) (:block/parent (first blocks))) blocks)]
+          (when (seq blocks)
+            (move-nodes blocks)))))))
 
 ;; selections
 (defn on-tab
   "direction = :left|:right, only indent or outdent when blocks are siblings"
   [direction]
-  (when-let [repo (state/get-current-repo)]
-    (let [blocks-dom-nodes (state/get-selection-blocks)
-          blocks (seq (reorder-selected-blocks blocks-dom-nodes))]
-      (when (seq blocks)
-        (let [end-node (get-top-level-end-node blocks)
-              end-node-parent (tree/-get-parent end-node)
-              top-level-nodes (->> (filter #(= (get-in end-node-parent [:data :db/id])
-                                               (get-in % [:block/parent :db/id])) blocks)
-                                   (map outliner-core/block))]
-          (outliner-core/indent-outdent-nodes top-level-nodes (= direction :right))
-          (let [opts {:key :block/change
-                      :data blocks}]
-            (db/refresh! repo opts)
-            (rehighlight-selected-nodes)))))))
+  (let [blocks-dom-nodes (state/get-selection-blocks)
+        blocks (seq (reorder-selected-blocks blocks-dom-nodes))]
+    (when (seq blocks)
+      (let [end-node (get-top-level-end-node blocks)
+            end-node-parent (tree/-get-parent end-node)
+            top-level-nodes (->> (filter #(= (get-in end-node-parent [:data :db/id])
+                                             (get-in % [:block/parent :db/id])) blocks)
+                                 (map outliner-core/block))]
+        (outliner-core/indent-outdent-nodes top-level-nodes (= direction :right))
+        (rehighlight-selected-nodes)))))
 
 (defn- get-link [format link label]
   (let [link (or link "")
@@ -2195,8 +2121,7 @@
                                    get-pos-fn
                                    page-block]
                             :as _opts}]
-  (let [repo (state/get-current-repo)
-        page (or page-block
+  (let [page (or page-block
                  (:block/page (db/entity (:db/id (state/get-edit-block)))))
         [target-block sibling? delete-editing-block? editing-block]
         ((or get-pos-fn get-block-tree-insert-pos-at-point))]
@@ -2223,9 +2148,7 @@
             _ (outliner-core/insert-nodes metadata-replaced-blocks target-block sibling?)
             _ (when (and delete-editing-block? editing-block)
                 (when-let [id (:db/id editing-block)]
-                  (outliner-core/delete-node (outliner-core/block (db/pull id)) true)))
-            new-blocks (db/pull-many repo '[*] (map (fn [id] [:block/uuid id]) @new-block-uuids))]
-        (db/refresh! repo {:key :block/insert :data new-blocks})
+                  (outliner-core/delete-node (outliner-core/block (db/pull id)) true)))]
         (last metadata-replaced-blocks)))))
 
 (defn- tree->vec-tree
@@ -2322,7 +2245,6 @@
                    opts)
              last-block (paste-block-vec-tree-at-target tree [:id :template :template-including-parent] opts)]
          (clear-when-saved!)
-         (db/refresh! repo {:key :block/insert :data [(db/pull db-id)]})
          (let [block (if (tree/satisfied-inode? last-block)
                        (:data last-block)
                        (:data (last (flatten last-block))))]
@@ -2342,9 +2264,7 @@
   [node]
   (when-not (parent-is-page? node)
     (let [parent-node (tree/-get-parent node)]
-      (outliner-core/move-subtree node parent-node true)))
-  (let [repo (state/get-current-repo)]
-    (db/refresh! repo {:key :block/change :data [(:data node)]})))
+      (outliner-core/move-subtree node parent-node true))))
 
 (defn- last-top-level-child?
   [{:keys [id]} current-node]
@@ -2881,10 +2801,7 @@
   (let [{:keys [block]} (get-state)]
     (when block
       (let [current-node (outliner-core/block block)]
-        (outliner-core/indent-outdent-nodes [current-node] indent?)
-        (let [repo (state/get-current-repo)]
-          (db/refresh! repo
-                       {:key :block/change :data [(:data current-node)]}))))
+        (outliner-core/indent-outdent-nodes [current-node] indent?)))
     (state/set-editor-op! :nil)))
 
 (defn keydown-tab-handler
@@ -3535,9 +3452,6 @@
                 (outliner-core/save-node block {:txs-state txs-state})))))))
       (let [block-id (first block-ids)
             input-pos (or (state/get-edit-pos) :max)]
-        (db/refresh! (state/get-current-repo)
-                    {:key :block/change
-                     :data [(db/pull [:block/uuid block-id])]})
         ;; update editing input content
         (when-let [editing-block (state/get-edit-block)]
           (when (= (:block/uuid editing-block) block-id)

src/main/frontend/handler/file.cljs

@@ -184,7 +184,7 @@
                            re-render-root? false
                            from-disk? false
                            skip-compare? false}}]
-  (let [original-content (db/get-file-no-sub repo path)
+  (let [original-content (db/get-file repo path)
         write-file! (if from-disk?
                       #(p/resolved nil)
                       #(fs/write-file! repo (config/get-repo-dir repo) path content
@@ -222,7 +222,7 @@
   ;; old file content
   (let [file->content (let [paths (map first files)]
                         (zipmap paths
-                                (map (fn [path] (db/get-file-no-sub repo path)) paths)))]
+                                (map (fn [path] (db/get-file repo path)) paths)))]
     ;; update db
     (when update-db?
       (doseq [[path content] files]
@@ -344,7 +344,7 @@
 (defn edn-file-set-key-value
   [path k v]
   (when-let [repo (state/get-current-repo)]
-    (when-let [content (db/get-file-no-sub path)]
+    (when-let [content (db/get-file path)]
       (common-handler/read-config content)
       (let [result (try
                      (rewrite/parse-string content)

src/main/frontend/handler/metadata.cljs

@@ -18,7 +18,7 @@
   (when-let [repo (state/get-current-repo)]
     (let [encrypted? (= k :db/encrypted-secret)
           path (config/get-metadata-path)
-          file-content (db/get-file-no-sub path)]
+          file-content (db/get-file path)]
       (p/let [_ (file-handler/create-metadata-file repo false)]
         (let [metadata-str (or file-content default-metadata-str)
               metadata (try

src/main/frontend/modules/editor/undo_redo.cljs

@@ -2,9 +2,7 @@
   (:require [datascript.core :as d]
             [frontend.db.conn :as conn]
             [frontend.modules.datascript-report.core :as db-report]
-            [frontend.db :as db]
             [frontend.state :as state]
-            [frontend.db.outliner :as db-outliner]
             [frontend.modules.outliner.pipeline :as pipelines]))
 
 ;;;; APIs
@@ -93,27 +91,17 @@
 
 ;;;; Invokes
 
-(defn get-by-id
-  [id]
-  (let [conn (conn/get-conn false)]
-    (db-outliner/get-by-id conn id)))
-
 (defn- transact!
   [txs]
   (let [conn (conn/get-conn false)
         db-report (d/transact! conn txs)]
     (pipelines/invoke-hooks db-report)))
 
-(defn- refresh!
-  [opts]
-  (let [repo (state/get-current-repo)]
-   (db/refresh! repo opts)))
-
 (defn undo
   []
   (let [[e prev-e] (pop-undo)]
     (when e
-      (let [{:keys [blocks txs]} e
+      (let [{:keys [txs]} e
             new-txs (get-txs false txs)
             editor-cursor (if (= (get-in e [:editor-cursor :last-edit-block :block/uuid])
                                  (get-in prev-e [:editor-cursor :last-edit-block :block/uuid])) ; same block
@@ -121,21 +109,16 @@
                             (:editor-cursor e))]
         (push-redo e)
         (transact! new-txs)
-        (let [blocks
-              (map (fn [x] (get-by-id [:block/uuid (:block/uuid x)])) blocks)]
-          (refresh! {:key :block/change :data (vec blocks)}))
         (assoc e
                :txs-op new-txs
                :editor-cursor editor-cursor)))))
 
 (defn redo
   []
-  (when-let [{:keys [blocks txs]:as e} (pop-redo)]
+  (when-let [{:keys [txs]:as e} (pop-redo)]
     (let [new-txs (get-txs true txs)]
       (push-undo e)
       (transact! new-txs)
-      (let [blocks (map (fn [x] (get-by-id [:block/uuid (:block/uuid x)])) blocks)]
-        (refresh! {:key :block/change :data (vec blocks)}))
       (assoc e :txs-op new-txs))))
 
 (defn listen-outliner-operation

src/main/frontend/modules/shortcut/data_helper.cljs

@@ -139,7 +139,7 @@
 (defn remove-shortcut [k]
   (let [repo (state/get-current-repo)
         path (cfg/get-config-path)]
-    (when-let [content (db/get-file-no-sub path)]
+    (when-let [content (db/get-file path)]
       (let [result (try
                      (rewrite/parse-string content)
                      (catch js/Error e

src/main/frontend/state.cljs

@@ -206,7 +206,8 @@
      :srs/mode?                             false
 
      :srs/cards-due-count                   nil
-     })))
+
+     :reactive/query-dbs                    {}})))
 
 ;; block uuid -> {content(String) -> ast}
 (def blocks-ast-cache (atom {}))
@@ -1149,13 +1150,6 @@
   (let [c (get-file-write-chan)]
     (count (gobj/get c "buf"))))
 
-(defn add-tx!
-  ;; TODO: replace f with data for batch transactions
-  [f]
-  (when f
-    (when-let [chan (get-db-batch-txs-chan)]
-      (async/put! chan f))))
-
 (defn get-left-sidebar-open?
   []
   (get-in @state [:ui/left-sidebar-open?]))
@@ -1621,6 +1615,19 @@
   [block-id]
   (sub [:ui/collapsed-blocks (get-current-repo) block-id]))
 
+(defn get-reactive-query-db
+  [ks]
+  (get-in @state [:reactive/query-dbs ks]))
+
+(defn set-reactive-query-db!
+  [ks db-value]
+  (when db-value
+    (set-state! [:reactive/query-dbs ks] db-value)))
+
+(defn delete-reactive-query-db!
+  [ks]
+  (update-state! :reactive/query-dbs (fn [dbs] (dissoc dbs ks))))
+
 (defn get-modal-id
   []
-  (:modal/id @state))
+  (:modal/id @state))

src/main/frontend/ui.cljs

@@ -364,13 +364,20 @@
     (reset! last-scroll-top scroll-top)
     down?))
 
-(defn on-scroll
-  [node {:keys [on-load on-top-reached threshold]
-         :or {threshold 500}}]
+(defn bottom-reached?
+  [node threshold]
   (let [full-height (gobj/get node "scrollHeight")
         scroll-top (gobj/get node "scrollTop")
-        client-height (gobj/get node "clientHeight")
-        bottom-reached? (<= (- full-height scroll-top client-height) threshold)
+        client-height (gobj/get node "clientHeight")]
+    (<= (- full-height scroll-top client-height) threshold)))
+
+(defn on-scroll
+  [node {:keys [on-load on-top-reached threshold bottom-reached]
+         :or {threshold 500}}]
+  (let [scroll-top (gobj/get node "scrollTop")
+        bottom-reached? (if (fn? bottom-reached)
+                          (bottom-reached)
+                          (bottom-reached? node threshold))
         top-reached? (= scroll-top 0)
         down? (scroll-down?)]
     (when (and down? bottom-reached? on-load)

src/main/frontend/util/marker.cljs

@@ -2,13 +2,6 @@
   (:require [clojure.string :as string]
             [frontend.util :as util]))
 
-(defn marker?
-  [s]
-  (contains?
-   #{"NOW" "LATER" "TODO" "DOING"
-     "DONE" "WAIT" "WAITING" "CANCELED" "CANCELLED" "STARTED" "IN-PROGRESS"}
-   (string/upper-case s)))
-
 (def marker-pattern
   #"^(NOW|LATER|TODO|DOING|DONE|WAITING|WAIT|CANCELED|CANCELLED|STARTED|IN-PROGRESS)?\s?")
 

src/main/frontend/util/page_property.cljs

@@ -1,7 +1,6 @@
 (ns frontend.util.page-property
   (:require [clojure.string :as string]
             [frontend.db :as db]
-            [frontend.handler.ui :as ui-handler]
             [frontend.modules.outliner.core :as outliner-core]
             [frontend.modules.outliner.file :as outliner-file]
             [frontend.state :as state]
@@ -68,9 +67,7 @@
               tx [(assoc page-id :block/properties new-properties)
                   block]]
           ;; (util/pprint tx)
-          (db/transact! tx)
-          (db/refresh! repo {:key :block/change
-                             :data [block]}))
+          (db/transact! tx))
         (let [block {:block/uuid (db/new-block-id)
                      :block/left page-id
                      :block/parent page-id
@@ -84,8 +81,5 @@
           (outliner-core/insert-node (outliner-core/block block)
                                      (outliner-core/block page)
                                      false)
-          (db/transact! [(assoc page-id :block/properties {key value})])
-          (db/refresh! repo {:key :block/change
-                             :data [block]})
-          (ui-handler/re-render-root!)))
+          (db/transact! [(assoc page-id :block/properties {key value})])))
       (outliner-file/sync-to-file page-id))))