fix: page unlinked refs

Use search indice to search first and then filter the result by the
db worker. This avoids using `d/datoms` which loads all the blocks to
the memory.

deps/db/src/logseq/db.cljs

@@ -548,7 +548,8 @@
                    distinct)
         refs (->> (mapcat (fn [id] (:block/_path-refs (d/entity db id))) alias)
                   distinct)]
-    (d/pull-many db '[*] (map :db/id refs))))
+    (when (seq refs)
+      (d/pull-many db '[*] (map :db/id refs)))))
 
 (defn get-block-refs
   [db id]
@@ -556,7 +557,8 @@
     (if (:block/name block)
       (get-page-refs db id)
       (let [refs (:block/_refs (d/entity db id))]
-       (d/pull-many db '[*] (map :db/id refs))))))
+        (when (seq refs)
+          (d/pull-many db '[*] (map :db/id refs)))))))
 
 (defn get-block-refs-count
   [db id]
@@ -564,6 +566,22 @@
           :block/_refs
           count))
 
+(defn get-page-unlinked-refs
+  "Get unlinked refs from search result"
+  [db page-id search-result-eids]
+  (let [alias (->> (get-page-alias db page-id)
+                   (cons page-id)
+                   set)
+        eids (remove
+              (fn [eid]
+                (when-let [e (d/entity db eid)]
+                  (or (some alias (map :db/id (:block/refs e)))
+                      (:block/link e)
+                      (nil? (:block/content e)))))
+              search-result-eids)]
+    (when (seq eids)
+      (d/pull-many db '[*] eids))))
+
 (comment
   (defn db-based-graph?
     "Whether the current graph is db-only"

src/main/frontend/components/block.cljs

@@ -3140,16 +3140,17 @@
         *navigating-block (get state ::navigating-block)
         navigating-block (rum/react *navigating-block)
         navigated? (and (not= (:block/uuid block) navigating-block) navigating-block)]
-    (when-not (state/sub-async-query-loading (:block/uuid block))
-      (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?})
-            config' (if original-block
-                      (assoc config :original-block original-block)
-                      config)
-            opts {}]
-        (rum/with-key
-          (block-container-inner state repo config' block
-                                 (merge opts {:navigating-block navigating-block :navigated? navigated?}))
-          (str "block-inner" (:block/uuid block)))))))
+    (when (:block/uuid block)
+      (when-not (state/sub-async-query-loading (:block/uuid block))
+        (let [[original-block block] (build-block config block {:navigating-block navigating-block :navigated? navigated?})
+              config' (if original-block
+                        (assoc config :original-block original-block)
+                        config)
+              opts {}]
+          (rum/with-key
+            (block-container-inner state repo config' block
+                                   (merge opts {:navigating-block navigating-block :navigated? navigated?}))
+            (str "block-inner" (:block/uuid block))))))))
 
 (defn divide-lists
   [[f & l]]

src/main/frontend/components/reference.cljs

@@ -17,7 +17,9 @@
             [frontend.util :as util]
             [rum.core :as rum]
             [frontend.modules.outliner.tree :as tree]
-            [frontend.db.async :as db-async]))
+            [frontend.db.async :as db-async]
+            [promesa.core :as p]
+            [frontend.search :as search]))
 
 (defn- frequencies-sort
   [references]
@@ -271,27 +273,26 @@
 
 (rum/defcs unlinked-references-aux
   < rum/reactive db-mixins/query
-  {:wrap-render
-   (fn [render-fn]
-     (fn [state]
-       (reset! (second (:rum/args state))
-               (apply +
-                      (for [[_ rfs]
-                            (db/get-page-unlinked-references
-                             (first (:rum/args state)))]
-                        (count rfs))))
-       (render-fn state)))}
+  {:init
+   (fn [state]
+     (let [*result (atom nil)
+           [page-name *n-ref] (:rum/args state)]
+       (p/let [result (search/get-page-unlinked-refs page-name)]
+         (reset! *n-ref (count result))
+         (reset! *result result))
+       (assoc state ::result *result)))}
   [state page-name _n-ref]
-  (let [ref-blocks (db/get-page-unlinked-references page-name)]
-    [:div.references-blocks
-     (let [ref-hiccup (block/->hiccup ref-blocks
-                                      {:id (str page-name "-unlinked-")
-                                       :ref? true
-                                       :group-by-page? true
-                                       :editor-box editor/box}
-                                      {})]
-       (content/content page-name
-                        {:hiccup ref-hiccup}))]))
+  (let [ref-blocks (rum/react (::result state))]
+    (when (seq ref-blocks)
+      [:div.references-blocks
+       (let [ref-hiccup (block/->hiccup ref-blocks
+                                        {:id (str page-name "-unlinked-")
+                                         :ref? true
+                                         :group-by-page? true
+                                         :editor-box editor/box}
+                                        {})]
+         (content/content page-name
+                          {:hiccup ref-hiccup}))])))
 
 (rum/defcs unlinked-references < rum/reactive
   (rum/local nil ::n-ref)

src/main/frontend/db.cljs

@@ -39,7 +39,7 @@
   get-files get-files-blocks get-files-full get-journals-length get-pages-with-file
   get-latest-journals get-page get-page-alias get-page-alias-names
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
-  get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages get-page-unlinked-references
+  get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages
   get-all-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages
   journal-page? page-alias-set sub-block
   set-file-last-modified-at! page-empty? page-exists? page-empty-or-dummy? get-alias-source-page

src/main/frontend/db/async.cljs

@@ -16,6 +16,7 @@
             [frontend.db.react :as react]))
 
 (def <q db-async-util/<q)
+(def <pull-many db-async-util/<pull-many)
 
 (defn <get-files
   [graph]
@@ -145,3 +146,8 @@
   (assert (integer? eid))
   (when-let [^Object worker @db-browser/*worker]
     (.get-block-refs-count worker graph eid)))
+
+(defn- pattern [name]
+  (re-pattern (str "(?i)(^|[^\\[#0-9a-zA-Z]|((^|[^\\[])\\[))"
+                   (util/regex-escape name)
+                   "($|[^0-9a-zA-Z])")))

src/main/frontend/db/async/util.cljs

@@ -7,7 +7,15 @@
 (defn <q
   [graph & inputs]
   (assert (not-any? fn? inputs) "Async query inputs can't include fns because fn can't be serialized")
-  (when-let [sqlite @state/*db-worker]
+  (when-let [^Object sqlite @state/*db-worker]
     (p/let [result (.q sqlite graph (pr-str inputs))]
       (when result
         (edn/read-string result)))))
+
+(defn <pull-many
+  [graph selector ids]
+  (assert (seq ids))
+  (when-let [^Object sqlite @state/*db-worker]
+    (p/let [result (.pull-many sqlite graph (pr-str selector) (pr-str ids))]
+      (when result
+        (edn/read-string result)))))

src/main/frontend/db/model.cljs

@@ -274,15 +274,7 @@ independent of format as format specific heading characters are stripped"
   [repo-url page]
   (when-let [page-id (:db/id (db-utils/entity repo-url [:block/name (util/safe-page-name-sanity-lc page)]))]
     (->>
-     (d/q '[:find ?e
-            :in $ ?page-name %
-            :where
-            [?page :block/name ?page-name]
-            (alias ?page ?e)]
-          (conn/get-db repo-url)
-          (util/safe-page-name-sanity-lc page)
-          (:alias rules/rules))
-     db-utils/seq-flatten
+     (ldb/get-page-alias (conn/get-db repo-url) page-id)
      (set)
      (set/union #{page-id}))))
 
@@ -863,37 +855,6 @@ independent of format as format specific heading characters are stripped"
                (sort-by-left-recursive)
                db-utils/group-by-page))))))
 
-(defn- pattern [name]
-  (re-pattern (str "(?i)(^|[^\\[#0-9a-zA-Z]|((^|[^\\[])\\[))"
-                   (util/regex-escape name)
-                   "($|[^0-9a-zA-Z])")))
-
-(defn get-page-unlinked-references
-  [page]
-  (when-let [repo (state/get-current-repo)]
-    (let [page (util/safe-page-name-sanity-lc page)
-          page-id     (:db/id (db-utils/entity [:block/name page]))
-          alias-names (get-page-alias-names repo page)
-          patterns    (->> (conj alias-names page)
-                           (map pattern))
-          filter-fn   (fn [datom]
-                        (some (fn [p]
-                                (re-find p (->> (:v datom)
-                                                (drawer/remove-logbook))))
-                              patterns))]
-      (->> (react/q repo [:frontend.worker.react/page-unlinked-refs page-id]
-             {:query-fn (fn [db _result]
-                          (let [ids
-                                (->> (d/datoms db :aevt :block/content)
-                                     (filter filter-fn)
-                                     (map :e))
-                                result (db-utils/pull-many repo block-attrs ids)]
-                            (remove (fn [block] (= page-id (:db/id (:block/page block)))) result)))}
-             nil)
-           react
-           (sort-by-left-recursive)
-           db-utils/group-by-page))))
-
 (defn get-block-referenced-blocks
   ([block-uuid]
    (get-block-referenced-blocks block-uuid {}))

src/main/frontend/db_worker.cljs

@@ -281,9 +281,18 @@
   (q [_this repo inputs-str]
      "Datascript q"
      (when-let [conn (worker-state/get-datascript-conn repo)]
-       (let [inputs (edn/read-string inputs-str)]
-         (let [result (apply d/q (first inputs) @conn (rest inputs))]
-           (pr-str result)))))
+       (let [inputs (edn/read-string inputs-str)
+             result (apply d/q (first inputs) @conn (rest inputs))]
+         (pr-str result))))
+
+  (pull-many
+   [_this repo selector-str ids-str]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [selector (edn/read-string selector-str)
+           ids (edn/read-string ids-str)
+           result (d/pull-many @conn selector ids)]
+       (let []
+         (pr-str result)))))
 
   (get-block-and-children
    [_this repo name children?]
@@ -301,6 +310,12 @@
    (when-let [conn (worker-state/get-datascript-conn repo)]
      (ldb/get-block-refs-count @conn id)))
 
+  (get-page-unlinked-refs
+   [_this repo page-id search-result-eids-str]
+   (when-let [conn (worker-state/get-datascript-conn repo)]
+     (let [search-result-eids (edn/read-string search-result-eids-str)]
+       (pr-str (ldb/get-page-unlinked-refs @conn page-id search-result-eids)))))
+
   (transact
    [_this repo tx-data tx-meta context]
    (when repo (worker-state/set-db-latest-tx-time! repo))

src/main/frontend/search.cljs

@@ -14,7 +14,11 @@
             [frontend.config :as config]
             [logseq.db.frontend.property :as db-property]
             [frontend.handler.file-based.property.util :as property-util]
-            [cljs-bean.core :as bean]))
+            [cljs-bean.core :as bean]
+            [frontend.db :as db]
+            [frontend.db.model :as db-model]
+            [frontend.db.utils :as db-utils]
+            [clojure.edn :as edn]))
 
 (def fuzzy-search fuzzy/fuzzy-search)
 
@@ -128,3 +132,22 @@
   [repo data]
   (when-let [engine (get-engine repo)]
     (protocol/transact-blocks! engine data)))
+
+(defn get-page-unlinked-refs
+  "Get matched result from search first, and then filter by worker db"
+  [page]
+  (when-let [repo (state/get-current-repo)]
+    (p/let [page-name (util/safe-page-name-sanity-lc page)
+            page (db/entity [:block/name page-name])
+            alias-names (conj (set (map util/safe-page-name-sanity-lc
+                                        (db/get-page-alias-names repo page-name))) page-name)
+            q (string/join " " alias-names)
+            result (block-search repo q {:limit 100})
+            eids (map (fn [b] [:block/uuid (:block/uuid b)]) result)
+            result (when (seq eids)
+                     (.get-page-unlinked-refs ^Object @state/*db-worker repo (:db/id page) (pr-str eids)))
+            result' (when result (edn/read-string result))]
+      (when result' (db/transact! repo result'))
+      (some->> result'
+               db-model/sort-by-left-recursive
+               db-utils/group-by-page))))