Lazy load page/block references and embeds

src/main/frontend/components/block.cljs

@@ -82,6 +82,7 @@
             [promesa.core :as p]
             [reitit.frontend.easy :as rfe]
             [rum.core :as rum]
+            [frontend.rum :as r]
             [shadow.loader :as loader]
             [logseq.common.path :as path]
             [electron.ipc :as ipc]
@@ -810,26 +811,34 @@
 (declare block-container)
 
 (rum/defc block-embed < rum/reactive
+  {:init (fn [state]
+           (let [block-id (second (:rum/args state))]
+             (db-async/<get-block-and-children (state/get-current-repo) block-id))
+           state)}
   [config uuid]
-  (when-let [block (db/entity [:block/uuid uuid])]
-    (let [repo (state/get-current-repo)]
-      (if (state/sub-block-unloaded? repo (str uuid))
-        [:span "Loading..."]
-        [:div.color-level.embed-block.bg-base-2
-         {:style {:z-index 2}
-          :on-double-click #(edit-parent-block % config)
-          :on-mouse-down (fn [e] (.stopPropagation e))}
-         [:div.px-3.pt-1.pb-2
-          (let [config' (assoc config
-                               :db/id (:db/id block)
-                               :id (str uuid)
-                               :embed-id uuid
-                               :embed? true
-                               :embed-parent (:block config)
-                               :ref? false)]
-            (blocks-container [block] config'))]]))))
+  (if (state/sub-block-unloaded? (str uuid))
+    [:span "Loading..."]
+    (when-let [block (db/entity [:block/uuid uuid])]
+      [:div.color-level.embed-block.bg-base-2
+       {:style {:z-index 2}
+        :on-double-click #(edit-parent-block % config)
+        :on-mouse-down (fn [e] (.stopPropagation e))}
+       [:div.px-3.pt-1.pb-2
+        (let [config' (assoc config
+                             :db/id (:db/id block)
+                             :id (str uuid)
+                             :embed-id uuid
+                             :embed? true
+                             :embed-parent (:block config)
+                             :ref? false)]
+          (blocks-container [block] config'))]])))
 
 (rum/defc page-embed < rum/reactive db-mixins/query
+  {:init (fn [state]
+           (let [page-name (second (:rum/args state))
+                 page-name' (util/page-name-sanity-lc (string/trim page-name))]
+             (db-async/<get-block-and-children (state/get-current-repo) page-name'))
+           state)}
   [config page-name]
   (let [page-name (util/page-name-sanity-lc (string/trim page-name))
         current-page (state/get-current-page)
@@ -841,22 +850,24 @@
      [:section.flex.items-center.p-1.embed-header
       [:div.mr-3 svg/page]
       (page-cp config {:block/name page-name})]
-     (when (and
-            (not= (util/page-name-sanity-lc (or current-page ""))
-                  page-name)
-            (not= (util/page-name-sanity-lc (get config :id ""))
-                  page-name))
-       (if whiteboard-page?
-         ((state/get-component :whiteboard/tldraw-preview) page-name)
-         (let [block (model/get-page page-name)
-               block (db/sub-block (:db/id block))
-               blocks (db/sort-by-left (:block/_parent block) block)]
-           (blocks-container blocks (assoc config
-                                           :db/id (:db/id block)
-                                           :id page-name
-                                           :embed? true
-                                           :page-embed? true
-                                           :ref? false)))))]))
+     (if (state/sub-block-unloaded? page-name)
+       [:span "Loading..."]
+       (when (and
+              (not= (util/page-name-sanity-lc (or current-page ""))
+                    page-name)
+              (not= (util/page-name-sanity-lc (get config :id ""))
+                    page-name))
+         (if whiteboard-page?
+           ((state/get-component :whiteboard/tldraw-preview) page-name)
+           (let [block (model/get-page page-name)
+                 block (db/sub-block (:db/id block))
+                 blocks (db/sort-by-left (:block/_parent block) block)]
+             (blocks-container blocks (assoc config
+                                             :db/id (:db/id block)
+                                             :id page-name
+                                             :embed? true
+                                             :page-embed? true
+                                             :ref? false))))))]))
 
 (defn- get-label-text
   [label]
@@ -880,88 +891,91 @@
 (declare breadcrumb)
 
 (rum/defc block-reference < rum/reactive
+  {:init (fn [state]
+           (let [block-id (second (:rum/args state))]
+             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+           state)}
   db-mixins/query
   [config id label]
   (if-let [block-id (if (uuid? id) id (parse-uuid id))]
-    (let [repo (state/get-current-repo)
-          block (db/entity [:block/uuid block-id])]
-      (if (state/sub-block-unloaded? repo (str block-id))
-        [:span "Loading..."]
-        (let [db-id (:db/id block)
-              block (when db-id (db/sub-block db-id))
-              properties (:block/properties block)
-              block-type (keyword (pu/lookup properties :ls-type))
-              hl-type (pu/lookup properties :hl-type)
-              repo (state/get-current-repo)
-              stop-inner-events? (= block-type :whiteboard-shape)]
-          (if (and block (:block/content block))
-            (let [title [:span.block-ref
-                         (block-content (assoc config :block-ref? true :stop-events? stop-inner-events?)
-                                        block nil (:block/uuid block)
-                                        (:slide? config)
-                                        false
-                                        (atom nil))]
-                  inner (if label
-                          (->elem
-                           :span.block-ref
-                           (map-inline config label))
-                          title)]
-              [:div.block-ref-wrap.inline
-               {:data-type    (name (or block-type :default))
-                :data-hl-type hl-type
-                :on-mouse-down
-                (fn [^js/MouseEvent e]
-                  (if (util/right-click? e)
-                    (state/set-state! :block-ref/context {:block (:block config)
-                                                          :block-ref block-id})
-                    (when (and
-                           (or (gobj/get e "shiftKey")
-                               (not (.. e -target (closest ".blank"))))
-                           (not (util/right-click? e)))
-                      (util/stop e)
-
-                      (cond
-                        (gobj/get e "shiftKey")
-                        (state/sidebar-add-block!
-                         (state/get-current-repo)
-                         (:db/id block)
-                         :block-ref)
-
-                        (and (util/meta-key? e) (whiteboard-handler/inside-portal? (.-target e)))
-                        (whiteboard-handler/add-new-block-portal-shape!
-                         (:block/uuid block)
-                         (whiteboard-handler/closest-shape (.-target e)))
-
-                        :else
-                        (match [block-type (util/electron?)]
+    (if (state/sub-block-unloaded? (str block-id))
+      [:span "Loading..."]
+      (let [block (db/entity [:block/uuid block-id])
+            db-id (:db/id block)
+            block (when db-id (db/sub-block db-id))
+            properties (:block/properties block)
+            block-type (keyword (pu/lookup properties :ls-type))
+            hl-type (pu/lookup properties :hl-type)
+            repo (state/get-current-repo)
+            stop-inner-events? (= block-type :whiteboard-shape)]
+        (if (and block (:block/content block))
+          (let [title [:span.block-ref
+                       (block-content (assoc config :block-ref? true :stop-events? stop-inner-events?)
+                                      block nil (:block/uuid block)
+                                      (:slide? config)
+                                      false
+                                      (atom nil))]
+                inner (if label
+                        (->elem
+                         :span.block-ref
+                         (map-inline config label))
+                        title)]
+            [:div.block-ref-wrap.inline
+             {:data-type    (name (or block-type :default))
+              :data-hl-type hl-type
+              :on-mouse-down
+              (fn [^js/MouseEvent e]
+                (if (util/right-click? e)
+                  (state/set-state! :block-ref/context {:block (:block config)
+                                                        :block-ref block-id})
+                  (when (and
+                         (or (gobj/get e "shiftKey")
+                             (not (.. e -target (closest ".blank"))))
+                         (not (util/right-click? e)))
+                    (util/stop e)
+
+                    (cond
+                      (gobj/get e "shiftKey")
+                      (state/sidebar-add-block!
+                       (state/get-current-repo)
+                       (:db/id block)
+                       :block-ref)
+
+                      (and (util/meta-key? e) (whiteboard-handler/inside-portal? (.-target e)))
+                      (whiteboard-handler/add-new-block-portal-shape!
+                       (:block/uuid block)
+                       (whiteboard-handler/closest-shape (.-target e)))
+
+                      :else
+                      (match [block-type (util/electron?)]
                           ;; pdf annotation
-                          [:annotation true] (pdf-assets/open-block-ref! block)
+                        [:annotation true] (pdf-assets/open-block-ref! block)
 
-                          [:whiteboard-shape true] (route-handler/redirect-to-whiteboard!
-                                                    (get-in block [:block/page :block/name]) {:block-id block-id})
+                        [:whiteboard-shape true] (route-handler/redirect-to-whiteboard!
+                                                  (get-in block [:block/page :block/name]) {:block-id block-id})
 
                           ;; default open block page
-                          :else (route-handler/redirect-to-page! id))))))}
-
-               (if (and (not (util/mobile?))
-                        (not (:preview? config))
-                        (not (:modal/show? @state/state))
-                        (nil? block-type))
-                 (ui/tippy {:html        (fn []
-                                           [:div.tippy-wrapper.overflow-y-auto.p-4
-                                            {:style {:width      735
-                                                     :text-align "left"
-                                                     :max-height 600}}
-                                            [(breadcrumb config repo block-id {:indent? true})
-                                             (blocks-container
-                                              (db/get-block-and-children repo block-id)
-                                              (assoc config :id (str id) :preview? true))]])
-                            :interactive true
-                            :in-editor?  true
-                            :delay       [1000, 100]} inner)
-                 inner)])
-            [:span.warning.mr-1 {:title "Block ref invalid"}
-             (block-ref/->block-ref id)]))))
+                        :else (route-handler/redirect-to-page! id))))))}
+
+             (if (and (not (util/mobile?))
+                      (not (:preview? config))
+                      (not (:modal/show? @state/state))
+                      (nil? block-type))
+               (ui/tippy {:html        (fn []
+                                         [:div.tippy-wrapper.overflow-y-auto.p-4
+                                          {:style {:width      735
+                                                   :text-align "left"
+                                                   :max-height 600}}
+                                          [(breadcrumb config repo block-id {:indent? true})
+                                           (blocks-container
+                                            (db/get-block-and-children repo block-id)
+                                            (assoc config :id (str id) :preview? true))]])
+                          :interactive true
+                          :in-editor?  true
+                          :delay       [1000, 100]} inner)
+               inner)])
+          [:span.warning.mr-1 {:title "Block ref invalid"}
+           (block-ref/->block-ref id)])))
 
     [:span.warning.mr-1 {:title "Block ref invalid"}
      (block-ref/->block-ref id)]))
@@ -2884,21 +2898,21 @@
 (defn- get-hidden-atom
   [sub-id *ref {:keys [initial-value]}]
   (let [*latest-value (atom nil)
-        *hidden? (rum/derived-atom [(:ui/main-container-scroll-top @state/state)] [::lazy-display sub-id]
-                   (fn [_top]
-                     (if (false? @*latest-value)
-                       @*latest-value
-                       (let [value (cond
-                                     (some? initial-value)
-                                     initial-value
-
-                                     @*ref
-                                     (hide-block? @*ref)
-
-                                     :else
-                                     true)]
-                         (reset! *latest-value value)
-                         value))))]
+        *hidden? (r/cached-derived-atom (:ui/main-container-scroll-top @state/state) [::lazy-display sub-id]
+                                        (fn [_top]
+                                          (if (false? @*latest-value)
+                                            @*latest-value
+                                            (let [value (cond
+                                                          (some? initial-value)
+                                                          initial-value
+
+                                                          @*ref
+                                                          (hide-block? @*ref)
+
+                                                          :else
+                                                          true)]
+                                              (reset! *latest-value value)
+                                              value))))]
     *hidden?))
 
 (rum/defcs ^:large-vars/cleanup-todo block-container-inner < rum/reactive db-mixins/query
@@ -3091,11 +3105,8 @@
   (rum/local false ::show-block-right-menu?)
   {:init (fn [state]
            (let [[config block] (:rum/args state)
-                 block-id (:block/uuid block)
-                 *loading? (atom true)]
-             (if (:block/content (db/entity [:block/uuid block-id]))
-               (reset! *loading? false)
-               (db-async/<get-block-and-children (state/get-current-repo) block-id *loading? :children? false))
+                 block-id (:block/uuid block)]
+             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false)
              (cond
                (root-block? config block)
                (state/set-collapsed-block! block-id false)
@@ -3107,7 +3118,6 @@
                :else
                nil)
              (assoc state
-                    ::loading? *loading?
                     ::control-show? (atom false)
                     ::navigating-block (atom (:block/uuid block)))))
    :will-unmount (fn [state]
@@ -3121,10 +3131,9 @@
   (let [repo (state/get-current-repo)
         *navigating-block (get state ::navigating-block)
         navigating-block (rum/react *navigating-block)
-        navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)
-        loading? (rum/react (::loading? state))]
+        navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)]
     (cond
-      loading?
+      (state/sub-block-unloaded? (:block/uuid block))
       [:div.ls-block.flex-1.flex-col.rounded-sm {:style {:width "100%"}}
        [:div.flex.flex-row
         [:div.flex.flex-row.items-center.mr-2.ml-1 {:style {:height 24}}

src/main/frontend/components/page.cljs

@@ -460,17 +460,12 @@
   (rum/local false ::hover-title?)
   {:init (fn [state]
            (let [page-name (:page-name (first (:rum/args state)))
-                 page-name' (get-sanity-page-name state page-name)
-                 entity (get-page-entity page-name')
-                 *loading? (atom true)]
-             (if (and entity (seq (:block/_left entity))) ; block and its children has been loaded
-               (reset! *loading? false)
-               (db-async/<get-block-and-children (state/get-current-repo) page-name' *loading?))
+                 page-name' (get-sanity-page-name state page-name)]
+             (db-async/<get-block-and-children (state/get-current-repo) page-name')
              (assoc state
-                    ::page-name page-name'
-                    ::loading? *loading?)))}
+                    ::page-name page-name')))}
   [state {:keys [repo page-name preview? sidebar?] :as option}]
-  (when-not (rum/react (::loading? state))
+  (when-not (state/sub-block-unloaded? (::page-name state))
     (when-let [path-page-name (get-path-page-name state page-name)]
       (let [current-repo (state/sub :git/current-repo)
             repo (or repo current-repo)
@@ -1029,6 +1024,7 @@
         *search-key (::search-key state)
         *search-input (rum/create-ref)
 
+        ;; TODO: remove this
         *indeterminate (rum/derived-atom
                         [*checks] ::indeterminate
                          (fn [checks]

src/main/frontend/components/property/value.cljs

@@ -20,7 +20,8 @@
             [rum.core :as rum]
             [frontend.handler.route :as route-handler]
             [frontend.handler.property.util :as pu]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [frontend.db.async :as db-async]))
 
 (defn- select-type?
   [property type]
@@ -443,9 +444,13 @@
                            :editor-box editor-box})])))
 
 (rum/defc property-template-value < rum/reactive
+  {:init (fn [state]
+           (let [block-id (second (:rum/args state))]
+             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+           state)}
   [config value opts]
   (when value
-    (if (state/sub-block-unloaded? (state/get-current-repo) value)
+    (if (state/sub-block-unloaded? value)
       [:div.text-sm.opacity-70 "loading"]
       (when-let [entity (db/sub-block (:db/id (db/entity [:block/uuid value])))]
         (let [properties-cp (:properties-cp opts)]
@@ -461,11 +466,15 @@
 
 (rum/defcs property-block-value < rum/reactive
   (rum/local nil ::template-instance)
+  {:init (fn [state]
+           (let [block-id (first (:rum/args state))]
+             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+           state)}
   [state value block property block-cp editor-box opts page-cp editor-id]
   (let [*template-instance (::template-instance state)
         template-instance @*template-instance]
     (when value
-      (if (state/sub-block-unloaded? (state/get-current-repo) value)
+      (if (state/sub-block-unloaded? value)
         [:div.text-sm.opacity-70 "loading"]
         (when-let [v-block (db/sub-block (:db/id (db/entity [:block/uuid value])))]
           (let [class? (contains? (:block/type v-block) "class")
@@ -493,9 +502,13 @@
               invalid-warning)))))))
 
 (rum/defc closed-value-item < rum/reactive
+  {:init (fn [state]
+           (let [block-id (first (:rum/args state))]
+             (db-async/<get-block-and-children (state/get-current-repo) block-id :children? false))
+           state)}
   [value {:keys [page-cp inline-text icon?]}]
   (when value
-    (if (state/sub-block-unloaded? (state/get-current-repo) value)
+    (if (state/sub-block-unloaded? value)
       [:div.text-sm.opacity-70 "loading"]
       (when-let [block (db/sub-block (:db/id (db/entity [:block/uuid value])))]
         (let [value' (get-in block [:block/schema :value])

src/main/frontend/db/async.cljs

@@ -12,7 +12,8 @@
             [frontend.db :as db]
             [frontend.persist-db.browser :as db-browser]
             [clojure.edn :as edn]
-            [datascript.core :as d]))
+            [datascript.core :as d]
+            [frontend.db.react :as react]))
 
 (def <q db-async-util/<q)
 
@@ -104,15 +105,28 @@
     (file-async/<get-file-based-property-values graph property)))
 
 (defn <get-block-and-children
-  [graph name-or-uuid &loading? & {:keys [children?]
-                                   :or {children? true}}]
-  (when-let [^Object sqlite @db-browser/*worker]
-    (p/let [name' (str name-or-uuid)
-            result (.get-block-and-children sqlite graph name' children?)
-            {:keys [block children] :as result'} (edn/read-string result)
-            conn (db/get-db graph false)
-            _ (d/transact! conn (cons block children))]
-      (reset! &loading? false)
-      (if children?
-        block
-        result'))))
+  [graph name-or-uuid & {:keys [children?]
+                         :or {children? true}}]
+  (let [name' (str name-or-uuid)
+        e (if (util/uuid-string? name')
+            (db/entity [:block/uuid (uuid name')])
+            (db/entity [:block/name name']))]
+    (cond
+      (and e (not children?) (:block/tx-id e))
+      e
+
+      (and e children? (:block/tx-id e) (seq (:block/_parent e)))
+      e
+
+      :else
+      (when-let [^Object sqlite @db-browser/*worker]
+        (state/update-state! :restore/unloaded-blocks (fn [s] (conj s name')))
+        (p/let [result (.get-block-and-children sqlite graph name' children?)
+                {:keys [block children] :as result'} (edn/read-string result)
+                conn (db/get-db graph false)
+                _ (d/transact! conn (cons block children))]
+          (state/update-state! :restore/unloaded-blocks (fn [s] (disj s name')))
+          (react/refresh-affected-queries! graph [[:frontend.worker.react/block (:db/id block)]])
+          (if children?
+            block
+            result'))))))

src/main/frontend/db/react.cljs

@@ -176,7 +176,7 @@
       (when-not (= new-result result)
        (set-new-result! k new-result)))))
 
-(defn- refresh-affected-queries!
+(defn refresh-affected-queries!
   [repo-url affected-keys]
   (util/profile
    "refresh!"

src/main/frontend/handler/events.cljs

@@ -74,6 +74,7 @@
             [promesa.core :as p]
             [lambdaisland.glogi :as log]
             [rum.core :as rum]
+            [frontend.rum :as r]
             [frontend.persist-db.browser :as db-browser]
             [frontend.db.rtc.debug-ui :as rtc-debug-ui]
             [frontend.modules.outliner.pipeline :as pipeline]
@@ -188,6 +189,9 @@
    state/set-state! :sync-graph/init? false))
 
 (defmethod handle :graph/switch [[_ graph opts]]
+  (state/set-state! :restore/unloaded-blocks #{})
+  (reset! r/*key->atom {})
+
   (let [^js sqlite @db-browser/*worker]
     (p/let [writes-finished? (when sqlite (.file-writes-finished? sqlite (state/get-current-repo)))
             request-finished? (ldb/request-finished?)]

src/main/frontend/rum.cljs

@@ -155,3 +155,12 @@
          #(js/document.removeEventListener event listener capture?)))
      [ref])
     set-ref))
+
+(defonce *key->atom (atom {}))
+(defn cached-derived-atom
+  "Make sure to return the same atom if `key` is the same."
+  [ref key f]
+  (or (get @*key->atom key)
+      (let [a (rum/derived-atom [ref] key f)]
+        (swap! *key->atom assoc key a)
+        a)))

src/main/frontend/state.cljs

@@ -18,7 +18,8 @@
             [logseq.common.config :as common-config]
             [medley.core :as medley]
             [promesa.core :as p]
-            [rum.core :as rum]))
+            [rum.core :as rum]
+            [frontend.rum :as r]))
 
 (defonce *profile-state
   (atom {}))
@@ -307,7 +308,8 @@
       :history/tx->editor-cursor             (atom {})
       :system/info                           {}
       ;; Whether block is selected
-      :ui/select-query-cache                 (atom {})})))
+      :ui/select-query-cache                 (atom {})
+      :restore/unloaded-blocks               (atom #{})})))
 
 ;; Block ast state
 ;; ===============
@@ -688,8 +690,8 @@ Similar to re-frame subscriptions"
   (let [*cache (:ui/select-query-cache @state)
         keys [::select-block block-uuid]
         atom (or (get @*cache keys)
-                 (let [result (rum/derived-atom
-                               [(:selection/blocks @state)]
+                 (let [result (r/cached-derived-atom
+                               (:selection/blocks @state)
                                keys
                                (fn [s]
                                  (contains? (set (get-selected-block-ids s)) block-uuid)))]
@@ -2302,11 +2304,11 @@ Similar to re-frame subscriptions"
   (storage/remove :user-groups))
 
 (defn sub-block-unloaded?
-  [repo block-uuid]
+  [block-uuid]
   (rum/react
-   (rum/derived-atom [(rum/cursor-in state [repo :restore/unloaded-blocks])] [::block-unloaded repo block-uuid]
-     (fn [s]
-       (contains? s (str block-uuid))))))
+   (r/cached-derived-atom (:restore/unloaded-blocks @state) [(get-current-repo) ::block-unloaded (str block-uuid)]
+                          (fn [s]
+                            (contains? s (str block-uuid))))))
 
 (defn get-color-accent []
   (get @state :ui/radix-color))