perf: long page rendering

src/main/frontend/components/block.cljs

@@ -4425,25 +4425,78 @@
          (when linked-block
            (str "-" (:block/uuid original-block))))))))
 
+(defn- blocks-source?
+  [blocks]
+  (and (map? blocks) (true? (:block-list/source? blocks))))
+
+(defn- blocks-total-count
+  [blocks]
+  (if (blocks-source? blocks)
+    (or (:total-count blocks) 0)
+    (count blocks)))
+
+(defn- blocks-item-at
+  [blocks idx]
+  (if (blocks-source? blocks)
+    (when-let [get-item (:get-item blocks)]
+      (get-item idx))
+    (nth blocks idx)))
+
+(defn- blocks-key-at
+  [blocks idx container-id]
+  (if (blocks-source? blocks)
+    (if-let [get-key (:get-key blocks)]
+      (get-key idx)
+      (str container-id "-idx-" idx))
+    (let [block (nth blocks idx)]
+      (str container-id "-" (:db/id block)))))
+
+(defn- blocks-ready?
+  [blocks idx]
+  (if (blocks-source? blocks)
+    (if-let [ready? (:ready? blocks)]
+      (boolean (ready? idx))
+      true)
+    true))
+
+(defn- blocks-placeholder
+  [blocks idx]
+  (if (blocks-source? blocks)
+    (if-let [placeholder (:placeholder blocks)]
+      (placeholder idx)
+      [:div {:style {:height 28}}])
+    [:div {:style {:height 28}}]))
+
 (rum/defc block-list
   [config blocks]
-  (let [[virtualized? _] (hooks/use-state (not (or (util/rtc-test?)
-                                                   (and (util/mobile?) (:journals? config))
-                                                   (if (:journals? config)
-                                                     (< (count blocks) 50)
-                                                     (< (count blocks) 10))
-                                                   (and (:block-children? config)
-                                                        ;; zoom-in block's children
-                                                        (not (and (:id config) (= (:id config) (str (:block/uuid (:block/parent (first blocks)))))))))))
+  (let [first-block (when-not (blocks-source? blocks) (first blocks))
+        total-count (blocks-total-count blocks)
+        disable-virtualized?
+        (or (util/rtc-test?)
+            (and (util/mobile?) (:journals? config)
+                 (if (:journals? config)
+                   (< total-count 50)
+                   (< total-count 10)))
+            (and (:block-children? config)
+                 ;; zoom-in block's children
+                 first-block
+                 (not (and (:id config)
+                           (= (:id config)
+                              (str (:block/uuid (:block/parent first-block))))))))
+        [virtualized? _] (hooks/use-state (not disable-virtualized?))
         render-item (fn [idx]
-                      (let [top? (zero? idx)
-                            bottom? (= (dec (count blocks)) idx)
-                            block (nth blocks idx)]
-                        (block-item (assoc config :top? top?)
-                                    block
-                                    {:top? top?
-                                     :bottom? bottom?})))
-        virtualized? (and virtualized? (seq blocks))
+                      (if-not (blocks-ready? blocks idx)
+                        (blocks-placeholder blocks idx)
+                        (let [top? (zero? idx)
+                              bottom? (= (dec total-count) idx)
+                              block (blocks-item-at blocks idx)]
+                          (if (some? block)
+                            (block-item (assoc config :top? top?)
+                                        block
+                                        {:top? top?
+                                         :bottom? bottom?})
+                            (blocks-placeholder blocks idx)))))
+        virtualized? (and (or (blocks-source? blocks) virtualized?) (pos? total-count))
         *virtualized-ref (hooks/use-ref nil)
         virtual-opts (when virtualized?
                        {:ref *virtualized-ref
@@ -4452,20 +4505,23 @@
                                                     (util/app-scroll-container-node node)
                                                     (util/app-scroll-container-node)))
                         :compute-item-key (fn [idx]
-                                            (let [block (nth blocks idx)]
-                                              (str (:container-id config) "-" (:db/id block))))
+                                            (blocks-key-at blocks idx (:container-id config)))
                         ;; Leave some space for the new inserted block
                         :increase-viewport-by 254
                         :overscan 254
-                        :total-count (count blocks)
+                        :total-count total-count
                         :item-content (fn [idx]
-                                        (let [top? (zero? idx)
-                                              bottom? (= (dec (count blocks)) idx)
-                                              block (nth blocks idx)]
-                                          (block-item (assoc config :top? top?)
-                                                      block
-                                                      {:top? top?
-                                                       :bottom? bottom?})))})
+                                        (render-item idx))
+                        :range-changed (when (blocks-source? blocks)
+                                         (when-let [on-range-changed (:on-range-changed blocks)]
+                                           (fn [r]
+                                             (let [start (cond
+                                                           (map? r) (or (get r :startIndex) (get r :start-index))
+                                                           :else (or (gobj/get r "startIndex") (.-startIndex r)))
+                                                   end (cond
+                                                         (map? r) (or (get r :endIndex) (get r :end-index))
+                                                         :else (or (gobj/get r "endIndex") (.-endIndex r)))]
+                                               (on-range-changed start end)))))})
         *wrap-ref (hooks/use-ref nil)]
     (hooks/use-effect!
      (fn []
@@ -4501,9 +4557,12 @@
        virtualized?
        (ui/virtualized-list virtual-opts)
        :else
-       (map-indexed (fn [idx block]
-                      (rum/with-key (render-item idx) (str (:container-id config) "-" (:db/id block))))
-                    blocks))]))
+       (if (blocks-source? blocks)
+         ;; Avoid non-virtualized render for block sources.
+         (ui/virtualized-list virtual-opts)
+         (map-indexed (fn [idx block]
+                        (rum/with-key (render-item idx) (str (:container-id config) "-" (:db/id block))))
+                      blocks)))]))
 
 (rum/defcs blocks-container < mixins/container-id rum/static
   {:init (fn [state]
@@ -4511,7 +4570,9 @@
   [state config blocks]
   (let [doc-mode? (:document/mode? config)
         id (::id state)]
-    (when (seq blocks)
+    (when (if (blocks-source? blocks)
+            (pos? (blocks-total-count blocks))
+            (seq blocks))
       [:div.blocks-container.flex-1
        {:id id
         :class (when doc-mode? "document-mode")

src/main/frontend/components/page.cljs

@@ -1,6 +1,7 @@
 (ns frontend.components.page
   (:require ["/frontend/utils" :as utils]
             [clojure.string :as string]
+            [datascript.core :as d]
             [dommy.core :as dom]
             [frontend.components.block :as component-block]
             [frontend.components.class :as class-component]
@@ -79,7 +80,7 @@
 
 (defn- open-root-block!
   [state]
-  (let [[_ block _ sidebar? preview?] (:rum/args state)]
+  (let [[block _ _ sidebar? preview?] (:rum/args state)]
     (when (and
            (or preview?
                (not (contains? #{:home :all-journals} (state/get-current-route))))
@@ -102,6 +103,76 @@
                            :sidebar? sidebar?})
          (str (:block/uuid page-e) "-hiccup"))])))
 
+(def ^:private long-page-direct-children-threshold 2000)
+
+(defn- direct-children-count
+  [db parent-eid]
+  (when (and db (integer? parent-eid))
+    (count (d/datoms db :avet :block/parent parent-eid))))
+
+(rum/defc long-page-blocks-inner
+  [page-e config sidebar? whiteboard? block-id]
+  (let [repo (state/get-current-repo)
+        parent-id (:db/id page-e)
+        [items set-items!] (hooks/use-state nil)
+        [ready-indices set-ready-indices!] (hooks/use-state #{})
+        *in-flight (hooks/use-ref #{})
+        ensure-loaded!
+        (fn [start end]
+          (when (seq items)
+            (let [total (count items)
+                  start (max 0 (int (or start 0)))
+                  end (min (dec total) (int (or end 0)))
+                  start (max 0 (- start 30))
+                  end (min (dec total) (+ end 30))
+                  in-flight (hooks/deref *in-flight)
+                  to-load (->> (range start (inc end))
+                               (remove ready-indices)
+                               (remove in-flight)
+                               vec)
+                  requests (mapv (fn [idx]
+                                   (let [{:keys [db/id block/uuid block/collapsed?]} (nth items idx)
+                                         collapsed? (if-some [v (state/get-block-collapsed uuid)]
+                                                      v
+                                                      collapsed?)]
+                                     {:id id
+                                      :opts {:children? (not collapsed?)}}))
+                                 to-load)]
+              (when (seq to-load)
+                (hooks/set-ref! *in-flight (into in-flight to-load))
+                (p/let [_ (db-async/<get-blocks-with-opts repo requests :skip-refresh? false)]
+                  (hooks/set-ref! *in-flight (reduce disj (hooks/deref *in-flight) to-load))
+                  (set-ready-indices! #(into % to-load)))))))]
+
+    (hooks/use-effect!
+     (fn []
+       (set-items! nil)
+       (set-ready-indices! #{})
+       (hooks/set-ref! *in-flight #{})
+       (p/let [items (db-async/<get-ordered-children-meta repo parent-id)]
+         (set-items! items)))
+     [repo parent-id])
+
+    (hooks/use-effect!
+     (fn []
+       (when (seq items)
+         (ensure-loaded! 0 30)))
+     [items])
+
+    (if-not (seq items)
+      [:div.page-blocks-inner {:style {:min-height 29}}]
+      (let [uuid->idx (into {} (map-indexed (fn [i m] [(:block/uuid m) i]) items))
+            source {:block-list/source? true
+                    :total-count (count items)
+                    :get-item (fn [idx]
+                                (when (contains? ready-indices idx)
+                                  (db/entity (:db/id (nth items idx)))))
+                    :get-key (fn [idx] (str (:block/uuid (nth items idx))))
+                    :ready? (fn [idx] (contains? ready-indices idx))
+                    :on-range-changed ensure-loaded!
+                    :index-of (fn [uuid] (get uuid->idx uuid))}]
+        (page-blocks-inner page-e source config sidebar? whiteboard? block-id)))))
+
 (declare page-cp)
 
 (rum/defc add-button-inner
@@ -185,6 +256,14 @@
                              [(take mobile-length-limit full-children) true]
                              [full-children false])
           quick-add-page-id (:db/id (db-db/get-built-in-page (db/get-db) common-config/quick-add-page-name))
+          long-page? (and
+                      (not block?)
+                      (not (:preview? config))
+                      (not config/publishing?)
+                      (not= id quick-add-page-id)
+                      (let [db (db/get-db)]
+                        (>= (or (direct-children-count db id) 0)
+                            long-page-direct-children-threshold)))
           children (cond
                      (and (= id quick-add-page-id)
                           (user-handler/user-uuid)
@@ -217,9 +296,19 @@
                               :document/mode? document-mode?}
                              config)
               config (common-handler/config-with-document-mode hiccup-config)
-              blocks (if block? [block] (db/sort-by-order children block))]
+              blocks (cond
+                       block?
+                       [block]
+
+                       long-page?
+                       nil
+
+                       :else
+                       (db/sort-by-order children block))]
           [:div.relative
-           (page-blocks-inner block blocks config sidebar? whiteboard? block-id)
+           (if long-page?
+             (long-page-blocks-inner block config sidebar? whiteboard? block-id)
+             (page-blocks-inner block blocks config sidebar? whiteboard? block-id))
            (when more?
              (shui/button {:variant :ghost
                            :class "text-muted-foreground w-full"

src/main/frontend/db/async.cljs

@@ -137,6 +137,55 @@
                   (js/console.error error)
                   (throw (ex-info "get-block error" {:block id-uuid-or-name}))))))))
 
+(defn <get-blocks-with-opts
+  "Batch version of `<get-block` that issues a single worker call.
+  `requests` is a seq of `{:id <db-id|uuid|page-name> :opts {...}}`."
+  [graph requests & {:keys [skip-transact? skip-refresh?]
+                     :or {skip-transact? false
+                          skip-refresh? false}}]
+  (let [requests (vec (remove nil? requests))]
+    (when (seq requests)
+      (p/catch
+       (p/let [result-transit-str (state/<invoke-db-worker-direct-pass
+                                   :thread-api/get-blocks
+                                   graph
+                                   (ldb/write-transit-str requests))
+               result (ldb/read-transit-str result-transit-str)]
+         (when-not skip-transact?
+           (let [conn (db/get-db graph false)
+                 block-and-children (->> result
+                                         (mapcat (fn [{:keys [block children]}]
+                                                   (cond-> []
+                                                     block (conj block)
+                                                     (seq children) (into children)))))
+                 tx-data (->> (remove (fn [b] (:block.temp/load-status (db/entity (:db/id b))))
+                                      block-and-children)
+                              (common-util/fast-remove-nils)
+                              (remove empty?))
+                 affected-keys (->> block-and-children
+                                    (keep :db/id)
+                                    (mapv (fn [id] [:frontend.worker.react/block id])))]
+             (when (seq tx-data)
+               (d/transact! conn tx-data))
+             (when-not skip-refresh?
+               (react/refresh-affected-queries! graph affected-keys {:skip-kv-custom-keys? true}))))
+         result)
+       (fn [error]
+         (js/console.error error)
+         (throw (ex-info "<get-blocks-with-opts error" {:requests requests} error)))))))
+
+(defn <get-ordered-children-meta
+  "Returns ordered direct children metadata (minimal maps with :db/id, :block/uuid, :block/collapsed?).
+  `parent-id` is the parent's db eid (page or block)."
+  [graph parent-id]
+  (when (integer? parent-id)
+    (p/let [result-transit-str (state/<invoke-db-worker-direct-pass
+                                :thread-api/get-ordered-children-meta
+                                graph
+                                (ldb/write-transit-str {:parent-id parent-id}))
+            result (ldb/read-transit-str result-transit-str)]
+      result)))
+
 (defn <get-blocks
   [graph ids* & {:as opts}]
   (let [ids (remove (fn [id] (:block.temp/load-status (db/entity id))) ids*)]

src/main/frontend/handler/ui.cljs

@@ -301,10 +301,15 @@
                         (when (util/uuid-string? id)
                           (uuid id))))]
       (when (and ref anchor-id)
-        (let [block-ids (map :block/uuid blocks)
-              find-idx (fn [anchor-id]
-                         (let [idx (.indexOf block-ids anchor-id)]
-                           (when (pos? idx) idx)))
+        (let [find-idx (cond
+                         (and (map? blocks) (true? (:block-list/source? blocks)) (fn? (:index-of blocks)))
+                         (:index-of blocks)
+
+                         :else
+                         (let [block-ids (to-array (map :block/uuid blocks))]
+                           (fn [anchor-id]
+                             (let [idx (.indexOf block-ids anchor-id)]
+                               (when (pos? idx) idx)))))
               idx (or (find-idx anchor-id)
                       (let [block (db/entity [:block/uuid anchor-id])
                             parents (map :block/uuid (db/get-block-parents (state/get-current-repo) (:block/uuid block) {}))]

src/main/frontend/worker/db_worker.cljs

@@ -427,6 +427,38 @@
   (let [requests (ldb/read-transit-str requests)]
     (get-blocks-with-cache repo requests)))
 
+(def ^:private *ordered-children-meta-cache (volatile! (cache/lru-cache-factory {} :threshold 200)))
+(def ^:private get-ordered-children-meta-with-cache
+  (common.cache/cache-fn
+   *ordered-children-meta-cache
+   (fn [repo parent-id]
+     (let [db (some-> (worker-state/get-datascript-conn repo) deref)]
+       [[repo (:max-tx db) parent-id]
+        [db parent-id]]))
+   (fn [db parent-id]
+     (when (and db (integer? parent-id))
+       (let [rows (d/q '[:find ?child ?order ?uuid ?collapsed
+                         :in $ ?parent
+                         :where
+                         [?child :block/parent ?parent]
+                         [?child :block/order ?order]
+                         [?child :block/uuid ?uuid]
+                         [(get-else $ ?child :block/collapsed? false) ?collapsed]]
+                       db
+                       parent-id)]
+         (->> rows
+              (sort-by (fn [[child-id order]] [order child-id]))
+              (mapv (fn [[child-id _order child-uuid collapsed?]]
+                      {:db/id child-id
+                       :block/uuid child-uuid
+                       :block/collapsed? collapsed?}))))))))
+
+(def-thread-api :thread-api/get-ordered-children-meta
+  [repo request]
+  (let [{:keys [parent-id]} (ldb/read-transit-str request)]
+    (when-let [items (get-ordered-children-meta-with-cache repo parent-id)]
+      (ldb/write-transit-str items))))
+
 (def-thread-api :thread-api/get-block-refs
   [repo id]
   (when-let [conn (worker-state/get-datascript-conn repo)]