perf: linked refs without filters

deps/db/src/logseq/db/common/initial_data.cljs

@@ -86,10 +86,10 @@
       block)))
 
 (defn get-block-children-ids
-  "Returns children UUIDs, notice the result doesn't include property value children ids."
-  [db block-uuid & {:keys [include-collapsed-children?]
-                    :or {include-collapsed-children? true}}]
-  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
+  "Returns children ids, notice the result doesn't include property value children ids."
+  [db block-eid & {:keys [include-collapsed-children?]
+                   :or {include-collapsed-children? true}}]
+  (when-let [eid (:db/id (d/entity db block-eid))]
     (let [seen (volatile! #{})]
       (loop [eids-to-expand [eid]]
         (when (seq eids-to-expand)
@@ -100,31 +100,30 @@
                                       (not (:block/collapsed? e))
                                       (common-entity-util/page? e))
                               (:block/_parent e)))) eids-to-expand)
-                uuids-to-add (keep :block/uuid children)]
-            (vswap! seen (partial apply conj) uuids-to-add)
+                ids-to-add (keep :db/id children)]
+            (vswap! seen (partial apply conj) ids-to-add)
             (recur (keep :db/id children)))))
       @seen)))
 
 (defn get-block-children
   "Including nested children, notice the result doesn't include property values."
-  {:arglists '([db block-uuid & {:keys [include-collapsed-children?]}])}
-  [db block-uuid & {:as opts}]
-  (let [ids (get-block-children-ids db block-uuid opts)]
+  {:arglists '([db eid & {:keys [include-collapsed-children?]}])}
+  [db eid & {:as opts}]
+  (let [ids (get-block-children-ids db eid opts)]
     (when (seq ids)
-      (map (fn [id] (d/entity db [:block/uuid id])) ids))))
+      (map (fn [id] (d/entity db id)) ids))))
 
 (defn get-block-full-children-ids
   "Including nested, collapsed and property value children."
-  {:arglists '([db block-uuid])}
-  [db block-uuid]
+  {:arglists '([db block-eid])}
+  [db block-eid]
   (d/q
    '[:find [?c ...]
      :in $ ?id %
      :where
-     [?p :block/uuid ?id]
-     (parent ?p ?c)]
+     (parent ?id ?c)]
    db
-   block-uuid
+   block-eid
    (:parent rules/rules)))
 
 (defn- with-raw-title
@@ -218,7 +217,7 @@
       ;; (prn :debug :get-block (:db/id block) (:block/title block) :children? children?
       ;;      :include-collapsed-children? include-collapsed-children?)
       (let [children (when children?
-                       (let [children-blocks (get-block-children db (:block/uuid block) {:include-collapsed-children? include-collapsed-children?})
+                       (let [children-blocks (get-block-children db (:db/id block) {:include-collapsed-children? include-collapsed-children?})
                              large-page? (>= (count children-blocks) 100)
                              children (let [children' (if large-page?
                                                         (:block/_parent block)

deps/db/src/logseq/db/common/reference.cljs

@@ -9,6 +9,7 @@
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.rules :as rules]))
 
 (defn get-filters
@@ -95,6 +96,7 @@
         (let [e (d/entity db eid)]
           (recur (:db/id (:block/parent e)) (conj parents' eid)))))))
 
+;; TODO(perf): recursive datascript rule is still too slow for filters for large graphs
 (defn get-linked-references
   [db id]
   (let [entity (d/entity db id)
@@ -102,28 +104,44 @@
         page-filters (get-filters db entity)
         excludes (map :db/id (:excluded page-filters))
         includes (map :db/id (:included page-filters))
+        has-filters? (or (seq excludes) (seq includes))
         class-ids (when (ldb/class? entity)
                     (let [class-children (db-class/get-structured-children db id)]
                       (set (conj class-children id))))
-        full-ref-block-ids (->> (mapcat (fn [id] (map :db/id (:block/_refs (d/entity db id)))) ids)
+        full-ref-block-ids (->> ids
+                                (mapcat (fn [id] (:block/_refs (d/entity db id))))
+                                (remove (fn [ref]
+                                          (or
+                                           (when class-ids
+                                             (some class-ids (map :db/id (:block/tags ref))))
+                                           (entity-util/hidden? ref)
+                                           (entity-util/hidden? (:block/page ref)))))
+                                (map :db/id)
                                 set)
-        matched-ref-block-ids (set (d/q (filter-refs-query includes excludes class-ids)
-                                        db
-                                        (rules/extract-rules rules/db-query-dsl-rules
-                                                             [:has-ref]
-                                                             {:deps rules/rules-dependencies})
-                                        ids))
-        matched-refs-with-children-ids (let [*result (atom #{})]
-                                         (doseq [ref-id matched-ref-block-ids]
-                                           (get-block-parents-until-top-ref db id ref-id full-ref-block-ids *result))
-                                         @*result)
-        ref-blocks (->> (set/intersection full-ref-block-ids matched-refs-with-children-ids)
+        matched-ref-block-ids (when has-filters?
+                                (let [ref-ids (d/q (filter-refs-query includes excludes class-ids)
+                                                   db
+                                                   (rules/extract-rules rules/db-query-dsl-rules
+                                                                        [:has-ref]
+                                                                        {:deps rules/rules-dependencies})
+                                                   ids)]
+                                  (set ref-ids)))
+        matched-refs-with-children-ids (when has-filters?
+                                         (let [*result (atom #{})]
+                                           (doseq [ref-id matched-ref-block-ids]
+                                             (get-block-parents-until-top-ref db id ref-id full-ref-block-ids *result))
+                                           @*result))
+        ref-blocks (->> (if has-filters?
+                          (set/intersection full-ref-block-ids matched-refs-with-children-ids)
+                          full-ref-block-ids)
                         (map (fn [id] (d/entity db id))))
-        filter-exists? (or (seq excludes) (seq includes))
-        children-ids (set (remove full-ref-block-ids matched-refs-with-children-ids))]
+        children-ids (if has-filters?
+                       (set (remove full-ref-block-ids matched-refs-with-children-ids))
+                       (->> (mapcat (fn [ref] (ldb/get-block-children-ids db (:db/id ref))) ref-blocks)
+                            set))]
     {:ref-blocks ref-blocks
      :ref-pages-count (get-ref-pages-count db id ref-blocks children-ids)
-     :ref-matched-children-ids (when filter-exists? children-ids)}))
+     :ref-matched-children-ids (when has-filters? children-ids)}))
 
 (defn get-unlinked-references
   [db id]

deps/outliner/src/logseq/outliner/core.cljs

@@ -409,7 +409,7 @@
         (swap! *txs-state concat [[:db/retract (:db/id block) :block/parent]
                                   [:db/retract (:db/id block) :block/order]
                                   [:db/retract (:db/id block) :block/page]])
-        (let [ids (cons (:db/id this) (ldb/get-block-full-children-ids db block-id))
+        (let [ids (cons (:db/id this) (ldb/get-block-full-children-ids db (:db/id block)))
               txs (map (fn [id] [:db.fn/retractEntity id]) ids)
               page-tx (let [block (d/entity db [:block/uuid block-id])]
                         (when (:block/pre-block? block)
@@ -979,7 +979,7 @@
                        (not (ldb/page? block))
                        (assoc :block/page target-page))]
             children-page-tx (when (and not-same-page? (not (ldb/page? block)))
-                               (let [children-ids (ldb/get-block-full-children-ids db (:block/uuid block))]
+                               (let [children-ids (ldb/get-block-full-children-ids db (:db/id block))]
                                  (keep (fn [id]
                                          (let [child (d/entity db id)]
                                            (when-not (ldb/page? child)

src/test/frontend/db/reference_test.cljs

@@ -67,7 +67,7 @@
   (d/transact! conn [[:db/retract foo-id :logseq.property.linked-references/includes]
                      [:db/retract foo-id :logseq.property.linked-references/excludes]]))
 
-(deftest ^:large-vars/cleanup-todo get-linked-references
+(deftest ^:large-vars/cleanup-todo ^:focus get-linked-references
   (let [conn (create-conn!)
         foo-id (:db/id (ldb/get-page @conn "foo"))
         _ (retract-filters! conn foo-id)