enhance(ux): render blocks by their orders (#11948)

* enhance: render nested children when rendering its parent

except that for long pages (500+ blocks).

* refactor: rename :block.temp/fully-loaded? to :block.temp/load-status

* fix: tests

* enhance: define keyword :block.temp/load-status

* update to :self

* chore: remove buggy parent cycle detect

* enhance(ux): don't delay rendering block properties

---------

Co-authored-by: rcmerci <[email protected]>

deps/db/src/logseq/db/common/entity_plus.cljc

@@ -22,7 +22,7 @@
     :block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker
 
     :block.temp/ast-title
-    :block.temp/fully-loaded? :block.temp/has-children? :block.temp/ast-body
+    :block.temp/load-status :block.temp/has-children? :block.temp/ast-body
 
     :db/valueType :db/cardinality :db/ident :db/index
 

deps/db/src/logseq/db/common/initial_data.cljs

@@ -1,7 +1,6 @@
 (ns logseq.db.common.initial-data
   "Provides db helper fns for graph initialization and lazy loading entities"
-  (:require [clojure.set :as set]
-            [datascript.core :as d]
+  (:require [datascript.core :as d]
             [datascript.impl.entity :as de]
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
@@ -86,10 +85,6 @@
       (update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
       block)))
 
-(defn- mark-block-fully-loaded
-  [b]
-  (assoc b :block.temp/fully-loaded? true))
-
 (comment
   (defn- property-without-db-attrs
     [property]
@@ -132,27 +127,30 @@
 
 (defn get-block-children-ids
   "Returns children UUIDs"
-  [db block-uuid]
+  [db block-uuid & {:keys [include-collapsed-children?]
+                    :or {include-collapsed-children? true}}]
   (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
-    (let [seen   (volatile! [])]
-      (loop [steps          100      ;check result every 100 steps
-             eids-to-expand [eid]]
+    (let [seen (volatile! #{})]
+      (loop [eids-to-expand [eid]]
         (when (seq eids-to-expand)
-          (let [eids-to-expand*
-                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
-                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
-            (when (and (zero? steps)
-                       (seq (set/intersection (set @seen) (set uuids-to-add))))
-              (throw (ex-info "bad outliner data, need to re-index to fix"
-                              {:seen @seen :eids-to-expand eids-to-expand})))
+          (let [children
+                (mapcat (fn [eid]
+                          (let [e (d/entity db eid)]
+                            (when (or include-collapsed-children?
+                                      (not (:block/collapsed? e))
+                                      (common-entity-util/page? e))
+
+                              (:block/_parent e)))) eids-to-expand)
+                uuids-to-add (keep :block/uuid children)]
             (vswap! seen (partial apply conj) uuids-to-add)
-            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
+            (recur (keep :db/id children)))))
       @seen)))
 
 (defn get-block-children
   "Including nested children."
-  [db block-uuid]
-  (let [ids (get-block-children-ids db block-uuid)]
+  {:arglists '([db block-uuid & {:keys [include-collapsed-children?]}])}
+  [db block-uuid & {:as opts}]
+  (let [ids (get-block-children-ids db block-uuid opts)]
     (when (seq ids)
       (map (fn [id] (d/entity db [:block/uuid id])) ids))))
 
@@ -163,10 +161,36 @@
     m))
 
 (defn- entity->map
-  [entity]
-  (-> (into {} entity)
-      (with-raw-title entity)
-      (assoc :db/id (:db/id entity))))
+  [entity & {:keys [level]
+             :or {level 0}}]
+  (let [opts {:level (inc level)}
+        f (if (> level 0)
+            identity
+            (fn [e]
+              (keep (fn [[k v]]
+                      (when-not (contains? #{:block/path-refs} k)
+                        (let [v' (cond
+                                   (= k :block/parent)
+                                   (:db/id v)
+                                   (= k :block/tags)
+                                   (set (map :db/id v))
+                                   (= k :logseq.property/created-by-ref)
+                                   (:db/id v)
+                                   (= k :block/refs)
+                                   (map #(select-keys % [:db/id :block/uuid :block/title]) v)
+                                   (de/entity? v)
+                                   (entity->map v opts)
+                                   (and (coll? v) (every? de/entity? v))
+                                   (map #(entity->map % opts) v)
+                                   :else
+                                   v)]
+                          [k v'])))
+                    e)))
+        m (->> (f entity)
+               (into {}))]
+    (-> m
+        (with-raw-title entity)
+        (assoc :db/id (:db/id entity)))))
 
 (defn hidden-ref?
   "Whether ref-block (for block with the `id`) should be hidden."
@@ -202,19 +226,9 @@
               count))
    0))
 
-(defn- update-entity->map
-  [m]
-  (update-vals m (fn [v]
-                   (cond
-                     (de/entity? v)
-                     (entity->map v)
-                     (and (coll? v) (every? de/entity? v))
-                     (map entity->map v)
-                     :else
-                     v))))
-
 (defn ^:large-vars/cleanup-todo get-block-and-children
-  [db id-or-page-name {:keys [children? children-only? nested-children? properties children-props]}]
+  [db id-or-page-name {:keys [children? properties include-collapsed-children?]
+                       :or {include-collapsed-children? false}}]
   (let [block (let [eid (cond (uuid? id-or-page-name)
                               [:block/uuid id-or-page-name]
                               (integer? id-or-page-name)
@@ -227,60 +241,29 @@
                   (d/entity db (get-first-page-by-name db (name id-or-page-name)))
                   :else
                   nil))
-        block-refs-count? (some #{:block.temp/refs-count} properties)
-        whiteboard? (common-entity-util/whiteboard? block)]
+        block-refs-count? (some #{:block.temp/refs-count} properties)]
     (when block
-      (let [children (when (or children? children-only?)
-                       (let [page? (common-entity-util/page? block)
-                             children (->>
-                                       (cond
-                                         (and nested-children? (not page?))
-                                         (get-block-children db (:block/uuid block))
-                                         nested-children?
-                                         (:block/_page block)
-                                         :else
-                                         (let [short-page? (when page?
-                                                             (<= (count (:block/_page block)) 100))]
-                                           (if short-page?
-                                             (:block/_page block)
-                                             (:block/_parent block))))
-                                       (remove (fn [e] (or (:logseq.property/created-from-property e)
-                                                           (:block/closed-value-property e)))))
-                             children-props (if whiteboard?
-                                              '[*]
-                                              (or children-props
-                                                  [:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
-                                                   ;; pre-loading feature-related properties to avoid UI refreshing
-                                                   :logseq.property/status :logseq.property.node/display-type :block/refs]))]
+      (let [children (when children?
+                       (let [children (let [children (get-block-children db (:block/uuid block) {:include-collapsed-children? include-collapsed-children?})
+                                            children' (if (>= (count children) 100)
+                                                        (:block/_parent block)
+                                                        children)]
+                                        (->> children'
+                                             (remove (fn [e] (:block/closed-value-property e)))))]
                          (map
                           (fn [block]
-                            (-> (if (= children-props '[*])
-                                  block
-                                  (-> (select-keys block children-props)
-                                      (with-raw-title block)
-                                      (assoc :block.temp/has-children? (some? (:block/_parent block)))))
-                                update-entity->map
-                                (dissoc :block/path-refs)))
-                          children)))]
-        (if children-only?
-          {:children children}
-          (let [block' (if (seq properties)
-                         (-> (select-keys block properties)
-                             (with-raw-title block)
-                             (assoc :db/id (:db/id block)))
-                         (entity->map block))
-                block' (cond->
-                        (mark-block-fully-loaded block')
-                         true
-                         update-entity->map
-                         true
-                         (dissoc :block/path-refs)
-                         block-refs-count?
-                         (assoc :block.temp/refs-count (get-block-refs-count db (:db/id block))))]
-            (cond->
-             {:block block'}
-              children?
-              (assoc :children children))))))))
+                            (-> block
+                                (assoc :block.temp/has-children? (some? (:block/_parent block)))
+                                (dissoc :block/tx-id :block/created-at :block/updated-at)
+                                entity->map))
+                          children)))
+            block' (cond-> (entity->map block)
+                     block-refs-count?
+                     (assoc :block.temp/refs-count (get-block-refs-count db (:db/id block))))]
+        (cond->
+         {:block block'}
+          children?
+          (assoc :children children))))))
 
 (defn get-latest-journals
   [db]

deps/db/src/logseq/db/frontend/validate.cljs

@@ -97,7 +97,7 @@
         ent-maps* (db-malli-schema/datoms->entities datoms)
         ent-maps (mapv
                   ;; Remove some UI interactions adding this e.g. import
-                  #(dissoc % :block.temp/fully-loaded? :block.temp/has-children?)
+                  #(dissoc % :block.temp/load-status :block.temp/has-children?)
                   (db-malli-schema/update-properties-in-ents db ent-maps*))
         errors (binding [db-malli-schema/*db-for-validate-fns* db]
                  (-> (map (fn [e]

deps/db/test/logseq/db_test.cljs

@@ -2,31 +2,24 @@
   (:require [cljs.test :refer [deftest is]]
             [datascript.core :as d]
             [logseq.db :as ldb]
-            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.test.helper :as db-test]))
 
 ;;; datoms
 ;;; - 1 <----+
 ;;;   - 2    |
 ;;;     - 3 -+
-(def broken-outliner-data-with-cycle
-  [{:db/id 1
-    :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
-    :block/parent 3}
-   {:db/id 2
-    :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
-    :block/parent 1}
-   {:db/id 3
-    :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
-    :block/parent 2}])
-
-(deftest get-block-children-ids-on-bad-outliner-data
-  (let [db (d/db-with (d/empty-db file-schema/schema)
-                      broken-outliner-data-with-cycle)]
-    (is (= "bad outliner data, need to re-index to fix"
-           (try (ldb/get-block-children-ids db #uuid "e538d319-48d4-4a6d-ae70-c03bb55b6fe4")
-                (catch :default e
-                  (ex-message e)))))))
+(comment
+  ;; TODO: throw error or fix broken data when cycle detected
+  (def broken-outliner-data-with-cycle
+    [{:db/id 1
+      :block/uuid #uuid"e538d319-48d4-4a6d-ae70-c03bb55b6fe4"
+      :block/parent 3}
+     {:db/id 2
+      :block/uuid #uuid"c46664c0-ea45-4998-adf0-4c36486bb2e5"
+      :block/parent 1}
+     {:db/id 3
+      :block/uuid #uuid"2b736ac4-fd49-4e04-b00f-48997d2c61a2"
+      :block/parent 2}]))
 
 (def class-parents-data
   [{:block/tags :logseq.class/Tag
@@ -82,4 +75,4 @@
         "Class pages correctly found for given class")
     (is (= nil
            (ldb/page-exists? @conn "movie" #{:logseq.class/Property}))
-        "Class pages correctly not found for given class")))
+        "Class pages correctly not found for given class")))

deps/outliner/src/logseq/outliner/core.cljs

@@ -240,7 +240,7 @@
           m* (cond->
               (-> data'
                   (dissoc :block/children :block/meta :block/unordered
-                          :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/fully-loaded?
+                          :block.temp/ast-title :block.temp/ast-body :block/level :block.temp/load-status
                           :block.temp/has-children?)
                   common-util/remove-nils
 

src/main/frontend/common_keywords.cljs

@@ -9,4 +9,9 @@
   :block/raw-title {:doc "like `:block/title`,
                           but when eval `(:block/raw-title block-entity)`, return raw title of this block"}
   :kv/value        {:doc "Used to store key-value, the value could be anything,
-                          e.g. {:db/ident :logseq.kv/xxx :kv/value value}"})
+                          e.g. {:db/ident :logseq.kv/xxx :kv/value value}"}
+
+
+  ;; :block.temp/xxx keywords
+  :block.temp/load-status {:doc "`:full` means the block and its children have been loaded,
+                                 `:self` means the block itself has been loaded."})

src/main/frontend/components/block.cljs

@@ -2642,8 +2642,7 @@
               (editor-handler/clear-selection!)
               (editor-handler/unhighlight-blocks!)
               (let [f #(p/do!
-                        (when-not (:block.temp/fully-loaded? (db/entity (:db/id block)))
-                          (db-async/<get-block (state/get-current-repo) (:db/id block) {:children? false}))
+                        (db-async/<get-block (state/get-current-repo) (:db/id block) {:children? false})
                         (let [cursor-range (some-> (gdom/getElement block-id)
                                                    (dom/by-class "block-content-inner")
                                                    first

src/main/frontend/components/page.cljs

@@ -696,7 +696,7 @@
                  *loading? (atom true)
                  page (db/get-page page-id-uuid-or-name)
                  *page (atom page)]
-             (when (:block.temp/fully-loaded? page) (reset! *loading? false))
+             (when (:block.temp/load-status page) (reset! *loading? false))
              (p/let [page-block (db-async/<get-block (state/get-current-repo) page-id-uuid-or-name)]
                (reset! *loading? false)
                (reset! *page (db/entity (:db/id page-block)))

src/main/frontend/components/views.cljs

@@ -909,7 +909,7 @@
 (rum/defc table-row < rum/reactive db-mixins/query
   [table row props option]
   (let [block (db/sub-block (:db/id row))
-        row' (if (:block.temp/fully-loaded? block)
+        row' (if (:block.temp/load-status block)
                (assoc block :block.temp/refs-count (:block.temp/refs-count row))
                row)]
     (table-row-inner table row' props option)))

src/main/frontend/db/async.cljs

@@ -80,11 +80,11 @@
                              (assoc opts :property-ident property-id))))
 
 (defn <get-block
-  [graph id-uuid-or-name & {:keys [children? nested-children? skip-transact? skip-refresh? children-only? properties]
+  [graph id-uuid-or-name & {:keys [children? skip-transact? skip-refresh? children-only? properties]
                             :or {children? true}
                             :as opts}]
 
-  ;; (prn :debug :<get-block id-uuid-or-name)
+  ;; (prn :debug :<get-block id-uuid-or-name :children? children?)
   ;; (js/console.trace)
   (let [name' (str id-uuid-or-name)
         opts (assoc opts :children? children?)
@@ -97,12 +97,11 @@
             (db/get-page name'))
         id (or (and (:block/uuid e) (str (:block/uuid e)))
                (and (util/uuid-string? name') name')
-               id-uuid-or-name)]
+               id-uuid-or-name)
+        load-status (:block.temp/load-status e)]
     (cond
-      (and (:block.temp/fully-loaded? e) ; children may not be fully loaded
-           (not children-only?)
-           (not children?)
-           (not nested-children?)
+      (and (or (= load-status :full)
+               (and (= load-status :self) (not children?) (not children-only?)))
            (not (some #{:block.temp/refs-count} properties)))
       (p/promise e)
 
@@ -113,11 +112,16 @@
                {:keys [block children]} (first result)]
          (when-not skip-transact?
            (let [conn (db/get-db graph false)
+                 load-status' (if (or children? children-only?) :full :self)
+                 block-load-status-tx (when block
+                                        [{:db/id (:db/id block)
+                                          :block.temp/load-status load-status'}])
                  block-and-children (if block (cons block children) children)
                  affected-keys [[:frontend.worker.react/block (:db/id block)]]
-                 tx-data (->> (remove (fn [b] (:block.temp/fully-loaded? (db/entity (:db/id b)))) block-and-children)
+                 tx-data (->> (remove (fn [b] (:block.temp/load-status (db/entity (:db/id b)))) block-and-children)
                               (common-util/fast-remove-nils)
-                              (remove empty?))]
+                              (remove empty?)
+                              (concat block-load-status-tx))]
              (when (seq tx-data) (d/transact! conn tx-data))
              (when-not skip-refresh?
                (react/refresh-affected-queries! graph affected-keys {:skip-kv-custom-keys? true}))))
@@ -129,7 +133,7 @@
 
 (defn <get-blocks
   [graph ids* & {:as opts}]
-  (let [ids (remove (fn [id] (:block.temp/fully-loaded? (db/entity id))) ids*)]
+  (let [ids (remove (fn [id] (:block.temp/load-status (db/entity id))) ids*)]
     (when (seq ids)
       (p/let [result (state/<invoke-db-worker :thread-api/get-blocks graph
                                               (map (fn [id]
@@ -138,7 +142,7 @@
         (let [conn (db/get-db graph false)
               result' (map :block result)]
           (when (seq result')
-            (let [result'' (map (fn [b] (assoc b :block.temp/fully-loaded? true)) result')]
+            (let [result'' (map (fn [b] (assoc b :block.temp/load-status :self)) result')]
               (d/transact! conn result'')))
           result')))))
 

src/main/frontend/extensions/pdf/assets.cljs

@@ -130,8 +130,8 @@
                 area? (not (nil? (:image content)))
                 wrap-props #(if-let [stamp (:image content)]
                               (assoc %
-                                :hl-type :area
-                                :hl-stamp stamp)
+                                     :hl-type :area
+                                     :hl-stamp stamp)
                               %)
                 props {:id (if (string? id) (uuid id) id)
                        (pu/get-pid :logseq.property/ls-type) :annotation
@@ -140,11 +140,11 @@
                 properties (wrap-props props)]
             (when (string? text)
               (editor-handler/api-insert-new-block!
-                text (merge {:page (:block/name ref-page)
-                             :custom-uuid id
-                             :edit-block? (not area?)
-                             :properties properties}
-                       insert-opts)))))))))
+               text (merge {:page (:block/name ref-page)
+                            :custom-uuid id
+                            :edit-block? (not area?)
+                            :properties properties}
+                           insert-opts)))))))))
 
 (defn db-based-ensure-ref-block!
   [pdf-current {:keys [id content page properties] :as hl} insert-opts]
@@ -187,8 +187,7 @@
   [hls-page]
   (p/let [result (db-async/<get-block (state/get-current-repo)
                                       (:block/uuid hls-page)
-                                      {:children? true
-                                       :nested-children? false})]
+                                      {:children? true})]
     {:highlights (keep :logseq.property.pdf/hl-value result)}))
 
 (defn file-based-load-hls-data$

src/main/frontend/handler/block.cljs

@@ -178,8 +178,7 @@
     (util/mobile-keep-keyboard-open)
     (let [repo (state/get-current-repo)]
       (p/do!
-       (when-not (:block.temp/fully-loaded? (db/entity (:db/id block)))
-         (db-async/<get-block repo (:db/id block) {:children? false}))
+       (db-async/<get-block repo (:db/id block) {:children? false})
        (when save-code-editor? (state/pub-event! [:editor/save-code-editor]))
        (when (not= (:block/uuid block) (:block/uuid (state/get-edit-block)))
          (state/clear-edit! {:clear-editing-block? false}))

src/main/frontend/handler/editor.cljs

@@ -2091,8 +2091,7 @@
                      (db-async/<get-template-by-name (name db-id)))
              block (when (:block/uuid block)
                      (db-async/<get-block repo (:block/uuid block)
-                                          {:children? true
-                                           :nested-children? true}))]
+                                          {:children? true}))]
        (when (:db/id block)
          (let [journal? (ldb/journal? target)
                target (or target (state/get-edit-block))
@@ -3486,7 +3485,7 @@
             repo (state/get-current-repo)
             result (db-async/<get-block repo (or block-id page-id)
                                         {:children-only? true
-                                         :nested-children? true})
+                                         :include-collapsed-children? true})
             blocks (if page-id
                      result
                      (cons (db/entity [:block/uuid block-id]) result))
@@ -3562,7 +3561,8 @@
 (defn expand-block! [block-id & {:keys [skip-db-collpsing?]}]
   (let [repo (state/get-current-repo)]
     (p/do!
-     (db-async/<get-block repo block-id {:children-only? true})
+     (db-async/<get-block repo block-id {:children-only? true
+                                         :include-collapsed-children? true})
      (when-not (or skip-db-collpsing? (skip-collapsing-in-db?))
        (set-blocks-collapsed! [block-id] false))
      (state/set-collapsed-block! block-id false))))

src/main/frontend/handler/events.cljs

@@ -422,8 +422,7 @@
     ;; load all nested children here for copy/export
     (p/all (map (fn [id]
                   (db-async/<get-block (state/get-current-repo) id
-                                       {:nested-children? true
-                                        :skip-refresh? false})) ids))))
+                                       {:skip-refresh? false})) ids))))
 
 (defn run!
   []

src/main/frontend/modules/outliner/pipeline.cljs

@@ -63,7 +63,7 @@
                             (if (contains? #{:create-property-text-block :insert-blocks} (:outliner-op tx-meta))
                               (let [update-blocks-fully-loaded (keep (fn [datom] (when (= :block/uuid (:a datom))
                                                                                    {:db/id (:e datom)
-                                                                                    :block.temp/fully-loaded? true})) tx-data)]
+                                                                                    :block.temp/load-status :self})) tx-data)]
                                 (concat update-blocks-fully-loaded tx-data))
                               tx-data))]
               (d/transact! conn tx-data' tx-meta))

src/main/frontend/util.cljc

@@ -814,7 +814,7 @@
                              (pr-str
                               {:graph graph
                                :embed-block? embed-block?
-                               :blocks (mapv #(dissoc % :block.temp/fully-loaded? %) blocks)}))}))]
+                               :blocks (mapv #(dissoc % :block.temp/load-status %) blocks)}))}))]
        (if owner-window
          (utils/writeClipboard data owner-window)
          (utils/writeClipboard data)))))

src/main/logseq/api.cljs

@@ -717,9 +717,7 @@
 (defn- <ensure-page-loaded
   [block-uuid-or-page-name]
   (p/let [repo (state/get-current-repo)
-          block (db-async/<get-block repo (str block-uuid-or-page-name)
-                                     {:children-props '[*]
-                                      :nested-children? true})
+          block (db-async/<get-block repo (str block-uuid-or-page-name) {})
           _ (when-let [page-id (:db/id (:block/page block))]
               (when-let [page-uuid (:block/uuid (db/entity page-id))]
                 (db-async/<get-block repo page-uuid)))]

src/main/logseq/api/block.cljs

@@ -149,7 +149,7 @@
                       ;; nested children results
                       (let [blocks (->> (db-model/get-block-and-children repo uuid)
                                         (map (fn [b]
-                                               (dissoc (db-utils/pull (:db/id b)) :block.temp/fully-loaded?))))]
+                                               (dissoc (db-utils/pull (:db/id b)) :block.temp/load-status))))]
                         (first (outliner-tree/blocks->vec-tree blocks uuid)))
                       ;; attached shallow children
                       (assoc block :block/children