fix: page/block refs and refs count

deps/db/src/logseq/db.cljs

@@ -528,6 +528,42 @@
                      '[:db/id :block/name :block/original-name]
                      ids)))))
 
+(defn get-page-alias
+  [db page-id]
+  (->>
+   (d/q
+    '[:find [?e ...]
+      :in $ ?page %
+      :where
+      (alias ?page ?e)]
+    db
+    page-id
+    (:alias rules/rules))
+   distinct))
+
+(defn get-page-refs
+  [db id]
+  (let [alias (->> (get-page-alias db id)
+                   (cons id)
+                   distinct)
+        refs (->> (mapcat (fn [id] (:block/_path-refs (d/entity db id))) alias)
+                  distinct)]
+    (d/pull-many db '[*] (map :db/id refs))))
+
+(defn get-block-refs
+  [db id]
+  (let [block (d/entity db id)]
+    (if (:block/name block)
+      (get-page-refs db id)
+      (let [refs (:block/_refs (d/entity db id))]
+       (d/pull-many db '[*] (map :db/id refs))))))
+
+(defn get-block-refs-count
+  [db id]
+  (some-> (d/entity db id)
+          :block/_refs
+          count))
+
 (comment
   (defn db-based-graph?
     "Whether the current graph is db-only"

src/main/frontend/components/block.cljs

@@ -816,7 +816,7 @@
              (db-async/<get-block-and-children (state/get-current-repo) block-id))
            state)}
   [config uuid]
-  (if (state/sub-block-unloaded? (str uuid))
+  (if (state/sub-async-query-loading (str uuid))
     [:span "Loading..."]
     (when-let [block (db/entity [:block/uuid uuid])]
       [:div.color-level.embed-block.bg-base-2
@@ -850,7 +850,7 @@
      [:section.flex.items-center.p-1.embed-header
       [:div.mr-3 svg/page]
       (page-cp config {:block/name page-name})]
-     (if (state/sub-block-unloaded? page-name)
+     (if (state/sub-async-query-loading page-name)
        [:span "Loading..."]
        (when (and
               (not= (util/page-name-sanity-lc (or current-page ""))
@@ -898,7 +898,7 @@
   db-mixins/query
   [config id label]
   (if-let [block-id (if (uuid? id) id (parse-uuid id))]
-    (if (state/sub-block-unloaded? (str block-id))
+    (if (state/sub-async-query-loading (str block-id))
       [:span "Loading..."]
       (let [block (db/entity [:block/uuid block-id])
             db-id (:db/id block)
@@ -2499,10 +2499,16 @@
                  current-block-page? (= (str (:block/uuid block)) (state/get-current-page))
                  embed-self? (and (:embed? config)
                                   (= (:block/uuid block) (:block/uuid (:block config))))
-                 default-hide? (not (and current-block-page? (not embed-self?) (state/auto-expand-block-refs?)))]
-             (assoc state ::hide-block-refs? (atom default-hide?))))}
+                 default-hide? (not (and current-block-page? (not embed-self?) (state/auto-expand-block-refs?)))
+                 *refs-count (atom nil)]
+             (p/let [count (db-async/<get-block-refs-count (state/get-current-repo) (:db/id block))]
+               (reset! *refs-count count))
+             (assoc state
+                    ::hide-block-refs? (atom default-hide?)
+                    ::refs-count *refs-count)))}
   [state config {:block/keys [uuid format] :as block} edit-input-id block-id edit? hide-block-refs-count? selected? *ref]
   (let [*hide-block-refs? (get state ::hide-block-refs?)
+        *refs-count (get state ::refs-count)
         hide-block-refs? (rum/react *hide-block-refs?)
         editor-box (get config :editor-box)
         editor-id (str "editor-" edit-input-id)
@@ -2527,7 +2533,7 @@
                                       :format format
                                       :on-hide (fn [value event]
                                                  (let [select? (and (= event :esc)
-                                                                      (not (string/includes? value "```")))]
+                                                                    (not (string/includes? value "```")))]
                                                    (p/let [_ (editor-handler/save-block! (editor-handler/get-state) value)]
                                                      (editor-handler/escape-editing select?))))}
                                      edit-input-id
@@ -2537,7 +2543,9 @@
              editor-cp
              (tags config block)]
             editor-cp))]
-       (let [refs-count (count (:block/_refs block))]
+       (let [refs-count (if (seq (:block/_refs block))
+                          (count (:block/_refs block))
+                          (rum/react *refs-count))]
          [:div.flex.flex-1.flex-col.block-content-wrapper
           [:div.flex.flex-row
            [:div.flex-1.w-full {:style {:display (if (:slide? config) "block" "flex")}}
@@ -2574,7 +2582,7 @@
               (block-refs-count block refs-count *hide-block-refs?)])]
 
           (when (and (not hide-block-refs?) (> refs-count 0))
-            (let [refs-cp (state/get-component :block/linked-references)]
+            (when-let [refs-cp (state/get-component :block/linked-references)]
               (refs-cp uuid)))]))]))
 
 (rum/defc single-block-cp
@@ -3132,7 +3140,7 @@
         *navigating-block (get state ::navigating-block)
         navigating-block (rum/react *navigating-block)
         navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)]
-    (when-not (state/sub-block-unloaded? (:block/uuid block))
+    (when-not (state/sub-async-query-loading (:block/uuid block))
       (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?})
             config' (if original-block
                       (assoc config :original-block original-block)

src/main/frontend/components/page.cljs

@@ -457,10 +457,9 @@
            (let [page-name (:page-name (first (:rum/args state)))
                  page-name' (get-sanity-page-name state page-name)]
              (db-async/<get-block-and-children (state/get-current-repo) page-name')
-             (assoc state
-                    ::page-name page-name')))}
+             (assoc state ::page-name page-name')))}
   [state {:keys [repo page-name preview? sidebar?] :as option}]
-  (when-not (state/sub-block-unloaded? (::page-name state))
+  (when-not (state/sub-async-query-loading (::page-name state))
     (when-let [path-page-name (get-path-page-name state page-name)]
       (let [current-repo (state/sub :git/current-repo)
             repo (or repo current-repo)
@@ -549,12 +548,13 @@
            (when-not block?
              (tagged-pages repo page-name page-original-name))
 
-       ;; referenced blocks
+           ;; referenced blocks
            (when-not block-or-whiteboard?
-             [:div {:key "page-references"}
-              (rum/with-key
-                (reference/references route-page-name)
-                (str route-page-name "-refs"))])
+             (when page
+               [:div {:key "page-references"}
+                (rum/with-key
+                  (reference/references route-page-name)
+                  (str route-page-name "-refs"))]))
 
            (when-not block-or-whiteboard?
              (when (not journal?)
@@ -577,7 +577,7 @@
            state)}
   [page]
   (when-let [repo (state/get-current-repo)]
-    (when-not (state/sub-block-unloaded? "contents")
+    (when-not (state/sub-async-query-loading "contents")
       (page-blocks-cp repo page {:sidebar? true}))))
 
 (defonce layout (atom [js/window.innerWidth js/window.innerHeight]))

src/main/frontend/components/property/value.cljs

@@ -450,7 +450,7 @@
            state)}
   [config value opts]
   (when value
-    (if (state/sub-block-unloaded? value)
+    (if (state/sub-async-query-loading value)
       [:div.text-sm.opacity-70 "loading"]
       (when-let [entity (db/sub-block (:db/id (db/entity [:block/uuid value])))]
         (let [properties-cp (:properties-cp opts)]
@@ -474,7 +474,7 @@
   (let [*template-instance (::template-instance state)
         template-instance @*template-instance]
     (when value
-      (if (state/sub-block-unloaded? value)
+      (if (state/sub-async-query-loading value)
         [:div.text-sm.opacity-70 "loading"]
         (when-let [v-block (db/sub-block (:db/id (db/entity [:block/uuid value])))]
           (let [class? (contains? (:block/type v-block) "class")
@@ -508,7 +508,7 @@
            state)}
   [value {:keys [page-cp inline-text icon?]}]
   (when value
-    (if (state/sub-block-unloaded? value)
+    (if (state/sub-async-query-loading value)
       [:div.text-sm.opacity-70 "loading"]
       (when-let [block (db/sub-block (:db/id (db/entity [:block/uuid value])))]
         (let [value' (get-in block [:block/schema :value])

src/main/frontend/components/reference.cljs

@@ -16,7 +16,8 @@
             [frontend.ui :as ui]
             [frontend.util :as util]
             [rum.core :as rum]
-            [frontend.modules.outliner.tree :as tree]))
+            [frontend.modules.outliner.tree :as tree]
+            [frontend.db.async :as db-async]))
 
 (defn- frequencies-sort
   [references]
@@ -96,23 +97,29 @@
     (filter-dialog-inner filters-atom *references page-name)))
 
 (rum/defc block-linked-references < rum/reactive db-mixins/query
+  {:init (fn [state]
+           (when-let [e (db/entity [:block/uuid (first (:rum/args state))])]
+             (db-async/<get-block-refs (state/get-current-repo) (:db/id e)))
+           state)}
   [block-id]
-  (let [e (db/entity [:block/uuid block-id])
-        page? (some? (:block/name e))
-        ref-blocks (if page?
-                     (-> (db/get-page-referenced-blocks (:block/name e))
-                         db-utils/group-by-page)
-                     (db/get-block-referenced-blocks block-id))
-        ref-hiccup (block/->hiccup ref-blocks
-                                   {:id (str block-id)
-                                    :ref? true
-                                    :breadcrumb-show? true
-                                    :group-by-page? true
-                                    :editor-box editor/box}
-                                   {})]
-    [:div.references-blocks
-     (content/content block-id
-                      {:hiccup ref-hiccup})]))
+  (when-let [e (db/entity [:block/uuid block-id])]
+    (when-not (state/sub-async-query-loading (str (:db/id e) "-refs"))
+      (let [page? (some? (:block/name e))
+            ref-blocks (if page?
+                         (-> (db/get-page-referenced-blocks (:block/name e))
+                             db-utils/group-by-page)
+                         (db/get-block-referenced-blocks block-id))]
+        (when (> (count ref-blocks) 0)
+          (let [ref-hiccup (block/->hiccup ref-blocks
+                                           {:id (str block-id)
+                                            :ref? true
+                                            :breadcrumb-show? true
+                                            :group-by-page? true
+                                            :editor-box editor/box}
+                                           {})]
+            [:div.references-blocks
+             (content/content block-id
+                              {:hiccup ref-hiccup})]))))))
 
 (rum/defc references-inner
   [page-name filters filtered-ref-blocks]
@@ -191,59 +198,65 @@
 (rum/defcs references* < rum/reactive db-mixins/query
   (rum/local nil ::ref-pages)
   {:init (fn [state]
-           (let [page-name (first (:rum/args state))
+           (let [page-name (->> (first (:rum/args state))
+                                util/page-name-sanity-lc)
+                 page (db/entity [:block/name page-name])
                  filters (when page-name (atom nil))]
+             (when page (db-async/<get-block-refs (state/get-current-repo) (:db/id page)))
              (assoc state ::filters filters)))}
   [state page-name]
   (when page-name
-    (let [page-name (util/page-name-sanity-lc page-name)
-          page-props-v (state/sub-page-properties-changed page-name)
-          *ref-pages (::ref-pages state)
-          repo (state/get-current-repo)
-          filters-atom (get state ::filters)
-          filter-state (rum/react filters-atom)
-          ref-blocks (db/get-page-referenced-blocks page-name)
-          page-id (:db/id (db/entity repo [:block/name page-name]))
-          aliases (db/page-alias-set repo page-name)
-          aliases-exclude-self (set (remove #{page-id} aliases))
-          top-level-blocks (filter (fn [b] (some aliases (set (map :db/id (:block/refs b))))) ref-blocks)
-          top-level-blocks-ids (set (map :db/id top-level-blocks))
-          filters (when (seq filter-state)
-                    (-> (group-by second filter-state)
-                        (update-vals #(map first %))))
-          filtered-ref-blocks (->> (block-handler/filter-blocks ref-blocks filters)
-                                   (block-handler/get-filtered-ref-blocks-with-parents ref-blocks))
-          total (count top-level-blocks)
-          filtered-top-blocks (filter (fn [b] (top-level-blocks-ids (:db/id b))) filtered-ref-blocks)
-          filter-n (count filtered-top-blocks)
-          parent->blocks (group-by (fn [x] (:db/id (x :block/parent))) filtered-ref-blocks)
-          result (->> (group-by :block/page filtered-top-blocks)
-                      (map (fn [[page blocks]]
-                             (let [blocks (sort-by (fn [b] (not= (:db/id page) (:db/id (:block/parent b)))) blocks)
-                                   result (map (fn [block]
-                                                 (let [filtered-children (get-filtered-children block parent->blocks)
-                                                       refs (when-not (contains? top-level-blocks-ids (:db/id (:block/parent block)))
-                                                              (block-handler/get-blocks-refed-pages aliases (cons block filtered-children)))
-                                                       block' (assoc (tree/block-entity->map block) :block/children filtered-children)]
-                                                   [block' refs])) blocks)
-                                   blocks' (map first result)
-                                   page' (if (contains? aliases-exclude-self (:db/id page))
-                                           {:db/id (:db/id page)
-                                            :block/alias? true
-                                            :block/journal-day (:block/journal-day page)}
-                                           page)]
-                               [[page' blocks'] (mapcat second result)]))))
-          filtered-ref-blocks' (map first result)
-          ref-pages (->>
-                     (mapcat second result)
-                     (map :block/original-name)
-                     frequencies)]
-      (reset! *ref-pages ref-pages)
-      (when (or (seq filter-state) (> filter-n 0))
-        [:div.references.page-linked.flex-1.flex-row
-         (sub-page-properties-changed page-name page-props-v filters-atom)
-         [:div.content.pt-6
-          (references-cp page-name filters filters-atom filter-state total filter-n filtered-ref-blocks' *ref-pages)]]))))
+    (let [repo (state/get-current-repo)
+          page-name (util/page-name-sanity-lc page-name)
+          page-entity (db/entity repo [:block/name page-name])]
+      (when page-entity
+        (when-not (state/sub-async-query-loading (str (:db/id page-entity) "-refs"))
+          (let [page-props-v (state/sub-page-properties-changed page-name)
+                *ref-pages (::ref-pages state)
+                filters-atom (get state ::filters)
+                filter-state (rum/react filters-atom)
+                ref-blocks (db/get-page-referenced-blocks page-name)
+                page-id (:db/id page-entity)
+                aliases (db/page-alias-set repo page-name)
+                aliases-exclude-self (set (remove #{page-id} aliases))
+                top-level-blocks (filter (fn [b] (some aliases (set (map :db/id (:block/refs b))))) ref-blocks)
+                top-level-blocks-ids (set (map :db/id top-level-blocks))
+                filters (when (seq filter-state)
+                          (-> (group-by second filter-state)
+                              (update-vals #(map first %))))
+                filtered-ref-blocks (->> (block-handler/filter-blocks ref-blocks filters)
+                                         (block-handler/get-filtered-ref-blocks-with-parents ref-blocks))
+                total (count top-level-blocks)
+                filtered-top-blocks (filter (fn [b] (top-level-blocks-ids (:db/id b))) filtered-ref-blocks)
+                filter-n (count filtered-top-blocks)
+                parent->blocks (group-by (fn [x] (:db/id (x :block/parent))) filtered-ref-blocks)
+                result (->> (group-by :block/page filtered-top-blocks)
+                            (map (fn [[page blocks]]
+                                   (let [blocks (sort-by (fn [b] (not= (:db/id page) (:db/id (:block/parent b)))) blocks)
+                                         result (map (fn [block]
+                                                       (let [filtered-children (get-filtered-children block parent->blocks)
+                                                             refs (when-not (contains? top-level-blocks-ids (:db/id (:block/parent block)))
+                                                                    (block-handler/get-blocks-refed-pages aliases (cons block filtered-children)))
+                                                             block' (assoc (tree/block-entity->map block) :block/children filtered-children)]
+                                                         [block' refs])) blocks)
+                                         blocks' (map first result)
+                                         page' (if (contains? aliases-exclude-self (:db/id page))
+                                                 {:db/id (:db/id page)
+                                                  :block/alias? true
+                                                  :block/journal-day (:block/journal-day page)}
+                                                 page)]
+                                     [[page' blocks'] (mapcat second result)]))))
+                filtered-ref-blocks' (map first result)
+                ref-pages (->>
+                           (mapcat second result)
+                           (map :block/original-name)
+                           frequencies)]
+            (reset! *ref-pages ref-pages)
+            (when (or (seq filter-state) (> filter-n 0))
+              [:div.references.page-linked.flex-1.flex-row
+               (sub-page-properties-changed page-name page-props-v filters-atom)
+               [:div.content.pt-6
+                (references-cp page-name filters filters-atom filter-state total filter-n filtered-ref-blocks' *ref-pages)]])))))))
 
 (rum/defc references
   [page-name]

src/main/frontend/components/whiteboard.cljs

@@ -48,7 +48,7 @@
         tldr (whiteboard-handler/page-name->tldr! page-name)
         generate-preview (when loaded?
                            (resolve 'frontend.extensions.tldraw/generate-preview))]
-    (when (and generate-preview (not (state/sub-block-unloaded? page-name)))
+    (when (and generate-preview (not (state/sub-async-query-loading page-name)))
       (generate-preview tldr))))
 
 ;; TODO: use frontend.ui instead of making a new one

src/main/frontend/db/async.cljs

@@ -120,13 +120,31 @@
 
       :else
       (when-let [^Object sqlite @db-browser/*worker]
-        (state/update-state! :restore/unloaded-blocks (fn [s] (conj s name')))
+        (state/update-state! :db/async-queries (fn [s] (conj s name')))
         (p/let [result (.get-block-and-children sqlite graph name' children?)
                 {:keys [block children] :as result'} (edn/read-string result)
                 conn (db/get-db graph false)
                 _ (d/transact! conn (cons block children))]
-          (state/update-state! :restore/unloaded-blocks (fn [s] (disj s name')))
+          (state/update-state! :db/async-queries (fn [s] (disj s name')))
           (react/refresh-affected-queries! graph [[:frontend.worker.react/block (:db/id block)]])
           (if children?
             block
             result'))))))
+
+(defn <get-block-refs
+  [graph eid]
+  (assert (integer? eid))
+  (when-let [^Object worker @db-browser/*worker]
+    (state/update-state! :db/async-queries (fn [s] (conj s (str eid "-refs"))))
+    (p/let [result-str (.get-block-refs worker graph eid)
+            result (edn/read-string result-str)
+            conn (db/get-db graph false)
+            _ (d/transact! conn result)]
+      (state/update-state! :db/async-queries (fn [s] (disj s (str eid "-refs"))))
+      result)))
+
+(defn <get-block-refs-count
+  [graph eid]
+  (assert (integer? eid))
+  (when-let [^Object worker @db-browser/*worker]
+    (.get-block-refs-count worker graph eid)))

src/main/frontend/db/model.cljs

@@ -813,8 +813,7 @@ independent of format as format specific heading characters are stripped"
          (->>
           (react/q repo
             [:frontend.worker.react/refs page-id]
-            {:use-cache? false
-             :query-fn (fn []
+            {:query-fn (fn []
                          (let [entities (mapcat (fn [id]
                                                   (:block/_path-refs (db-utils/entity id))) pages)
                                blocks (map (fn [e]

src/main/frontend/db_worker.cljs

@@ -291,6 +291,16 @@
    (when-let [conn (worker-state/get-datascript-conn repo)]
      (pr-str (sqlite-common-db/get-block-and-children @conn name children?))))
 
+  (get-block-refs
+   [_this repo id]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (pr-str (ldb/get-block-refs @conn id))))
+
+  (get-block-refs-count
+   [_this repo id]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (ldb/get-block-refs-count @conn id)))
+
   (transact
    [_this repo tx-data tx-meta context]
    (when repo (worker-state/set-db-latest-tx-time! repo))

src/main/frontend/extensions/tldraw.cljs

@@ -207,7 +207,7 @@
              (db-async/<get-block-and-children (state/get-current-repo) page-name)
              state))}
   [page-name block-id loaded-app set-loaded-app]
-  (when-not (state/sub-block-unloaded? page-name)
+  (when-not (state/sub-async-query-loading page-name)
     (let [populate-onboarding? (whiteboard-handler/should-populate-onboarding-whiteboard? page-name)
           on-mount (fn [^js tln]
                      (when tln

src/main/frontend/handler/events.cljs

@@ -189,7 +189,7 @@
    state/set-state! :sync-graph/init? false))
 
 (defmethod handle :graph/switch [[_ graph opts]]
-  (state/set-state! :restore/unloaded-blocks #{})
+  (state/set-state! :db/async-queries #{})
   (reset! r/*key->atom {})
 
   (let [^js sqlite @db-browser/*worker]

src/main/frontend/state.cljs

@@ -311,7 +311,7 @@
       :system/info                           {}
       ;; Whether block is selected
       :ui/select-query-cache                 (atom {})
-      :restore/unloaded-blocks               (atom #{})})))
+      :db/async-queries               (atom #{})})))
 
 ;; Block ast state
 ;; ===============
@@ -2305,12 +2305,12 @@ Similar to re-frame subscriptions"
   []
   (storage/remove :user-groups))
 
-(defn sub-block-unloaded?
-  [block-uuid]
+(defn sub-async-query-loading
+  [k]
+  (assert (some? k))
   (rum/react
-   (r/cached-derived-atom (:restore/unloaded-blocks @state) [(get-current-repo) ::block-unloaded (str block-uuid)]
-                          (fn [s]
-                            (contains? s (str block-uuid))))))
+   (r/cached-derived-atom (:db/async-queries @state) [(get-current-repo) ::async-query (str k)]
+                          (fn [s] (contains? s (str k))))))
 
 (defn get-color-accent []
   (get @state :ui/radix-color))