handle linked refs and unlinked refs in worker instead of ui thread

deps/db/src/logseq/db.cljs

@@ -18,7 +18,6 @@
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.rules :as rules]
-            [logseq.db.frontend.view :as db-view]
             [logseq.db.sqlite.util :as sqlite-util])
   (:refer-clojure :exclude [object?]))
 
@@ -463,23 +462,9 @@
           :block/_refs
           count))
 
-(defn get-page-unlinked-refs
-  "Get unlinked refs from search result"
-  [db page-id search-result-eids]
-  (let [alias (->> (get-block-alias db page-id)
-                   (cons page-id)
-                   set)
-        eids (remove
-              (fn [eid]
-                (when-let [e (d/entity db eid)]
-                  (or (some alias (map :db/id (:block/refs e)))
-                      (:block/link e)
-                      (nil? (:block/title e)))))
-              search-result-eids)]
-    (when (seq eids)
-      (d/pull-many db '[*] eids))))
-
-(def hidden-or-internal-tag? db-view/hidden-or-internal-tag?)
+(defn hidden-or-internal-tag?
+  [e]
+  (or (entity-util/hidden? e) (db-class/internal-tags (:db/ident e))))
 
 (defn get-all-pages
   [db]
@@ -614,5 +599,3 @@
     nil))
 
 (def get-recent-updated-pages sqlite-common-db/get-recent-updated-pages)
-
-(def get-view-data db-view/get-view-data)

deps/db/src/logseq/db/frontend/view.cljs

@@ -5,6 +5,7 @@
             [datascript.core :as d]
             [datascript.impl.entity :as de]
             [logseq.common.util :as common-util]
+            [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
@@ -48,10 +49,6 @@
           (when (valid-type? v)
             v))))))
 
-(defn hidden-or-internal-tag?
-  [e]
-  (or (entity-util/hidden? e) (db-class/internal-tags (:db/ident e))))
-
 (defn- by [sorting]
   (fn [a b]
     (loop [[{:keys [asc? get-value]} & orderings] sorting]
@@ -206,6 +203,39 @@
            result)))
      (:filters filters))))
 
+(defn- get-linked-references
+  [db id]
+  (let [entity (d/entity db id)
+        ids (set (cons id (ldb/get-block-alias db id)))
+        refs (mapcat (fn [id] (:block/_refs (d/entity db id))) ids)]
+    (->> refs
+         (remove (fn [block]
+                   (or
+                    (= (:db/id block) id)
+                    (= id (:db/id (:block/page block)))
+                    (ldb/hidden? (:block/page block))
+                    (contains? (set (map :db/id (:block/tags block))) (:db/id entity))
+                    (some? (get block (:db/ident entity))))))
+         (common-util/distinct-by :db/id))))
+
+(defn- get-unlinked-references
+  [db id]
+  (let [entity (d/entity db id)
+        title (string/lower-case (:block/title entity))]
+    (when-not (string/blank? title)
+      (let [ids (->> (d/datoms db :avet :block/title)
+                     (keep (fn [d]
+                             (when (and (not= id (:e d)) (string/includes? (string/lower-case (:v d)) title))
+                               (:e d)))))]
+        (keep
+         (fn [eid]
+           (let [e (d/entity db eid)]
+             (when-not (or (some #(= id %) (map :db/id (:block/refs e)))
+                           (:block/link e)
+                           (ldb/page? e))
+               e)))
+         ids)))))
+
 (defn- get-entities
   [db view feat-type property-ident]
   (let [view-for (:logseq.property/view-for view)
@@ -216,7 +246,7 @@
       :all-pages
       (keep (fn [d]
               (let [e (d/entity db (:e d))]
-                (when-not (or (hidden-or-internal-tag? e)
+                (when-not (or (ldb/hidden-or-internal-tag? e)
                               (entity-util/property? e)
                               (entity-util/built-in? e))
                   e)))
@@ -239,41 +269,45 @@
                              {:deps rules/rules-dependencies})
         property-ident)
        (keep (fn [id] (non-hidden-e id))))
+      :linked-references
+      (get-linked-references db (:db/id view-for))
+      :unlinked-references
+      (get-unlinked-references db (:db/id view-for))
+      :query-result
+      nil
       nil)))
 
 (defonce *view-cache (atom {}))
 (defn get-view-data
   [repo db view-id]
-  (time
-   (let [view (d/entity db view-id)
-         feat-type (:logseq.property.view/feature-type view)
-         index-attr (case feat-type
-                      :all-pages
-                      :block/name
-                      :class-objects
-                      :block/tags
-                      :property-objects
-                      (let [view-for (:logseq.property/view-for view)]
-                        (:db/ident view-for))
-                      nil)
-         filters (:logseq.property.table/filters view)]
-     (when index-attr
-       (let [data (if-let [cache (get-in @*view-cache [repo view-id])]
-                    cache
-                    (let [entities (get-entities db view feat-type index-attr)
-                          sorting (let [sorting* (:logseq.property.table/sorting view)]
-                                    (if (or (= sorting* :logseq.property/empty-placeholder) (empty? sorting*))
-                                      [{:id :block/updated-at, :asc? false}]
-                                      sorting*))
+  (let [view (d/entity db view-id)
+        feat-type (:logseq.property.view/feature-type view)
+        index-attr (case feat-type
+                     :all-pages
+                     :block/name
+                     :class-objects
+                     :block/tags
+                     :property-objects
+                     (let [view-for (:logseq.property/view-for view)]
+                       (:db/ident view-for))
+                     nil)
+        filters (:logseq.property.table/filters view)
+        data (if-let [cache (get-in @*view-cache [repo view-id])]
+               cache
+               (let [entities (get-entities db view feat-type index-attr)
+                     sorting (let [sorting* (:logseq.property.table/sorting view)]
+                               (if (or (= sorting* :logseq.property/empty-placeholder) (empty? sorting*))
+                                 [{:id :block/updated-at, :asc? false}]
+                                 sorting*))
 
-                          result (->>
-                                   ;; filter
-                                  (cond->> entities
-                                    (seq filters)
-                                    (filter (fn [row] (row-matched? db row filters))))
+                     result (->>
+                                  ;; filter
+                             (cond->> entities
+                               (seq filters)
+                               (filter (fn [row] (row-matched? db row filters))))
                                    ;; sort
-                                  (sort-rows db sorting))]
-                      (swap! *view-cache assoc-in [repo view-id] result)
-                      result))]
-         {:count (count data)
-          :full-block-ids (mapv :db/id data)})))))
+                             (sort-rows db sorting))]
+                 (swap! *view-cache assoc-in [repo view-id] result)
+                 result))]
+    {:count (count data)
+     :data (mapv :db/id data)}))

src/main/frontend/components/all_pages.cljs

@@ -36,14 +36,11 @@
 
 (rum/defc all-pages < rum/static
   []
-  (let [[data set-data!] (rum/use-state nil)
-        columns' (views/build-columns {} (columns)
+  (let [columns' (views/build-columns {} (columns)
                                       {:with-object-name? false
                                        :with-id? false})]
     [:div.ls-all-pages.w-full.mx-auto
-     (views/view {:data data
-                  :set-data! set-data!
-                  :view-parent (db/get-page common-config/views-page-name)
+     (views/view {:view-parent (db/get-page common-config/views-page-name)
                   :view-feature-type :all-pages
                   :show-items-count? true
                   :columns columns'

src/main/frontend/components/objects.cljs

@@ -45,8 +45,7 @@
 
 (rum/defc class-objects-inner < rum/static
   [config class properties]
-  (let [[data set-data!] (rum/use-state nil)
-        ;; Properties can be nil for published private graphs
+  (let [;; Properties can be nil for published private graphs
         properties' (remove nil? properties)
         columns* (views/build-columns config properties' {:add-tags-column? true})
         columns (cond
@@ -63,12 +62,10 @@
                   columns)]
 
     (views/view {:config config
-                 :data data
-                 :set-data! set-data!
                  :view-parent class
                  :view-feature-type :class-objects
                  :columns columns
-                 :add-new-object! (fn [{:keys [properties]}]
+                 :add-new-object! (fn [{:keys [properties set-data!]}]
                                     (if (= :logseq.class/Asset (:db/ident class))
                                       (shui/dialog-open!
                                        (fn []
@@ -78,8 +75,9 @@
                                            {:on-change (fn [_e files]
                                                          (p/do!
                                                           (editor-handler/upload-asset! nil files :markdown editor-handler/*asset-uploading? true)
-                                                          (set-data! (get-class-objects class))
-                                                          (shui/dialog-close!)))})]))
+                                                          (shui/dialog-close!)
+                                                          ;; FIXME: set-data!
+                                                          ))})]))
                                       (add-new-class-object! class set-data! properties)))
                  :show-add-property? true
                  :show-items-count? true
@@ -134,15 +132,12 @@
 
 (rum/defc property-related-objects-inner < rum/static
   [config property properties]
-  (let [[data set-data!] (rum/use-state nil)
-        columns (views/build-columns config properties)]
+  (let [columns (views/build-columns config properties)]
     (views/view {:config config
-                 :data data
                  :view-parent property
                  :view-feature-type :property-objects
-                 :set-data! set-data!
                  :columns columns
-                 :add-new-object! (fn [{:keys [properties]}]
+                 :add-new-object! (fn [{:keys [properties set-data!]}]
                                     (add-new-property-object! property set-data! properties))
                  ;; TODO: Add support for adding column
                  :show-add-property? false
@@ -152,7 +147,6 @@
                                      (let [pages (->> selected-rows (filter entity-util/page?) (remove :logseq.property/built-in?))
                                            blocks (->> selected-rows (remove entity-util/page?) (remove :logseq.property/built-in?))]
                                        (p/do!
-                                        (set-data! (get-property-related-objects (state/get-current-repo) property))
                                         (when-let [f (get-in table [:data-fns :set-row-selection!])]
                                           (f {}))
                                         (ui-outliner-tx/transact!

src/main/frontend/components/reference.cljs

@@ -44,111 +44,37 @@
              (content/content block-id
                               {:hiccup ref-hiccup})]))))))
 
-(rum/defc references-inner
-  [page-entity filters filtered-ref-blocks]
-  (let [*ref (rum/use-ref nil)]
-    [:div.references-blocks.faster.fade-in {:ref *ref}
-     (let [ref-hiccup (block/->hiccup filtered-ref-blocks
-                                      {:id (str (:block/uuid page-entity))
-                                       :ref? true
-                                       :breadcrumb-show? true
-                                       :group-by-page? true
-                                       :editor-box editor/box
-                                       :filters filters}
-                                      {})]
-       (content/content (str (:block/uuid page-entity)) {:hiccup ref-hiccup}))]))
-
-(defn- columns
-  [config result]
-  (->> (mapcat :block.temp/property-keys result)
-       distinct
-       (map db/entity)
-       (ldb/sort-by-order)
-       ((fn [cs] (views/build-columns config cs {:add-tags-column? false})))))
-
 (rum/defc references-cp
   [page-entity *filters total filter-n filtered-ref-blocks *ref-pages]
   (let [filters @*filters
-        *collapsed? (atom nil)
-        db-based? (config/db-based-graph?)
-        reference-filter (if db-based?
-                           (shui/button
-                            {:title "Page filter"
-                             :variant "ghost"
-                             :class "text-muted-foreground !px-1"
-                             :size :sm
-                             :on-click (fn [e]
-                                         (shui/popup-show! (.-target e)
-                                                           (fn []
-                                                             [:div.p-4
-                                                              (filters/filter-dialog page-entity *filters *ref-pages)])
-                                                           {:align "end"}))}
-                            (ui/icon "filter-cog"
-                                     {:class (cond
-                                               (and (empty? (:included filters)) (empty? (:excluded filters)))
-                                               ""
-
-                                               (and (seq (:included filters)) (empty? (:excluded filters)))
-                                               "text-success"
-
-                                               (and (empty? (:included filters)) (seq (:excluded filters)))
-                                               "text-error"
-                                               :else
-                                               "text-warning")}))
-                           [:a.filter.fade-link
-                            {:title (t :linked-references/filter-heading)
-                             :on-mouse-over (fn [_e]
-                                              (when @*collapsed? ; collapsed
-                           ;; expand
-                                                (reset! @*collapsed? false)))
-                             :on-pointer-down (fn [e]
-                                                (util/stop-propagation e)
-                                                (shui/popup-show! (.-target e)
-                                                                  (fn []
-                                                                    [:div.p-4
-                                                                     (filters/filter-dialog page-entity *filters *ref-pages)])
-                                                                  {:align "end"}))}
-                            (ui/icon "filter" {:class (cond
-                                                        (and (empty? (:included filters)) (empty? (:excluded filters)))
-                                                        "opacity-60 hover:opacity-100"
-
-                                                        (and (seq (:included filters)) (empty? (:excluded filters)))
-                                                        "text-success"
-
-                                                        (and (empty? (:included filters)) (seq (:excluded filters)))
-                                                        "text-error"
-                                                        :else
-                                                        "text-warning")
-                                               :size  22})])]
-    (if db-based?
-      (let [blocks (->> (mapcat second filtered-ref-blocks)
-                        (map (fn [b] (assoc (db/entity (:db/id b)) :id (:db/id b)))))
-            columns' (columns {} blocks)]
-        (when (or (seq blocks)
-                  (seq (:included filters))
-                  (seq (:excluded filters)))
-          (views/view
-           {:view-parent page-entity
-            :view-feature-type :linked-references
-            :additional-actions [reference-filter]
-            :data blocks
-            :columns columns'})))
-      (let [threshold (state/get-linked-references-collapsed-threshold)
-            default-collapsed? (or (>= total threshold) (ldb/class? page-entity))]
-        (ui/foldable
-         [:div.flex.flex-row.flex-1.justify-between.items-center
-          [:div.font-medium.opacity-50
-           (t :linked-references/reference-count (when (or (seq (:included filters))
-                                                           (seq (:excluded filters))) filter-n) total)]
-          reference-filter]
-
-         (fn []
-           (references-inner page-entity filters filtered-ref-blocks))
-
-         {:default-collapsed? default-collapsed?
-          :title-trigger? true
-          :init-collapsed (fn [collapsed-atom]
-                            (reset! *collapsed? collapsed-atom))})))))
+        reference-filter (shui/button
+                          {:title "Page filter"
+                           :variant "ghost"
+                           :class "text-muted-foreground !px-1"
+                           :size :sm
+                           :on-click (fn [e]
+                                       (shui/popup-show! (.-target e)
+                                                         (fn []
+                                                           [:div.p-4
+                                                            (filters/filter-dialog page-entity *filters *ref-pages)])
+                                                         {:align "end"}))}
+                          (ui/icon "filter-cog"
+                                   {:class (cond
+                                             (and (empty? (:included filters)) (empty? (:excluded filters)))
+                                             ""
+
+                                             (and (seq (:included filters)) (empty? (:excluded filters)))
+                                             "text-success"
+
+                                             (and (empty? (:included filters)) (seq (:excluded filters)))
+                                             "text-error"
+                                             :else
+                                             "text-warning")}))]
+    (views/view
+     {:view-parent page-entity
+      :view-feature-type :linked-references
+      :additional-actions [reference-filter]
+      :columns (views/build-columns {} [] {})})))
 
 (defn- get-filtered-children
   [block parent->blocks]
@@ -238,37 +164,12 @@
    (references* entity)))
 
 (rum/defcs unlinked-references-aux
-  < rum/reactive db-mixins/query
-  {:init
-   (fn [state]
-     (let [*result (atom nil)
-           [page *n-ref] (:rum/args state)]
-       (p/let [result (search/get-unlinked-refs (:db/id page))]
-         (reset! *n-ref (count result))
-         (reset! *result result))
-       (assoc state ::result *result)))}
   [state page _n-ref]
-  (let [ref-blocks (rum/react (::result state))]
-    (when (seq ref-blocks)
-      (if (config/db-based-graph?)
-        (let [blocks (->> (mapcat val ref-blocks)
-                          (map (fn [b] (assoc (db/entity (:db/id b)) :id (:db/id b)))))
-              columns' (columns {} blocks)]
-          (views/view
-           {:view-parent page
-            :view-feature-type :unlinked-references
-            :data blocks
-            :columns columns'
-            :foldable-options {:default-collapsed? true}}))
-        [:div.references-blocks
-         (let [ref-hiccup (block/->hiccup ref-blocks
-                                          {:id (str (:block/title page) "-unlinked-")
-                                           :ref? true
-                                           :group-by-page? true
-                                           :editor-box editor/box}
-                                          {})]
-           (content/content (:block/name page)
-                            {:hiccup ref-hiccup}))]))))
+  (views/view
+   {:view-parent page
+    :view-feature-type :unlinked-references
+    :columns (views/build-columns {} [] {})
+    :foldable-options {:default-collapsed? true}}))
 
 (rum/defcs unlinked-references < rum/reactive
   (rum/local nil ::n-ref)

src/main/frontend/components/views.cljs

@@ -1086,8 +1086,8 @@
        (property-handler/set-block-property! repo (:db/id entity) :logseq.property.table/sized-columns sized-columns))}))
 
 (rum/defc lazy-item
-  [idx {:keys [full-block-ids properties]} item-render]
-  (let [db-id (util/nth-safe full-block-ids idx)
+  [data idx {:keys [properties]} item-render]
+  (let [db-id (util/nth-safe data idx)
         block (db/entity db-id)
         [item set-item!] (hooks/use-state block)
         opts {:children? false
@@ -1116,7 +1116,7 @@
     :skipAnimationFrameInResizeObserver true
     :total-count (or (:items-count option) (count rows))
     :item-content (fn [idx]
-                    (lazy-item idx option
+                    (lazy-item (:data table) idx option
                                (fn [row]
                                  (table-row table row {} option))))
     :items-rendered (fn [props]
@@ -1179,8 +1179,9 @@
          :total-count (count blocks)
          :custom-scroll-parent (gdom/getElement "main-content-container")
          :item-content (fn [idx]
-                         (when-let [block (nth blocks idx)]
-                           (gallery-card-item table view-entity block config')))}))]))
+                         (lazy-item (:data table) idx {}
+                                    (fn [block]
+                                      (gallery-card-item table view-entity block config'))))}))]))
 
 (defn- run-effects!
   [option {:keys [data state data-fns]} input input-filters set-input-filters! *scroller-ref gallery?]
@@ -1598,7 +1599,7 @@
         [views set-views!] (hooks/use-state nil)
         [items-count set-count!] (hooks/use-state (count (:data option)))
         [loading? set-loading!] (hooks/use-state true)
-        [full-block-ids set-full-block-ids!] (hooks/use-state [])]
+        [data set-data!] (hooks/use-state [])]
     (hooks/use-effect!
      (fn []
        (let [repo (state/get-current-repo)]
@@ -1616,9 +1617,9 @@
                                      (set-views! (concat views [new-view]))
                                      new-view)))]
             (if (and current-view
-                     (nil? (:data option)))
-              (p/let [{:keys [count full-block-ids]} (<load-view-data current-view)]
-                (set-full-block-ids! full-block-ids)
+                     (empty? (:data option)))
+              (p/let [{:keys [count data]} (<load-view-data current-view)]
+                (set-data! data)
                 (set-count! count)
                 (set-loading! false))
               (set-loading! false)))
@@ -1631,8 +1632,8 @@
        (repeat 3 (shui/skeleton {:class "h-6 w-full"}))]
       [:div.flex.flex-col.gap-2
        (view-container view-entity (assoc option
+                                          :data data
                                           :items-count items-count
-                                          :full-block-ids full-block-ids
                                           :views views
                                           :set-views! set-views!
                                           :set-view-entity! set-view-entity!))])))

src/main/frontend/worker/db_worker.cljs

@@ -35,6 +35,7 @@
             [logseq.db.common.order :as db-order]
             [logseq.db.common.sqlite :as sqlite-common-db]
             [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.frontend.view :as db-view]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.sqlite.util :as sqlite-util]
@@ -554,12 +555,6 @@
                         (map (fn [b] (d/pull @conn '[*] (:db/id b)))))]
        (ldb/write-transit-str parents))))
 
-  (get-page-unlinked-refs
-   [_this repo page-id search-result-eids-str]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [search-result-eids (ldb/read-transit-str search-result-eids-str)]
-       (ldb/write-transit-str (ldb/get-page-unlinked-refs @conn page-id search-result-eids)))))
-
   (set-context
    [_this context]
    (let [context (if (string? context)
@@ -917,7 +912,7 @@
   (get-view-data
    [_this repo view-id opts-str]
    (let [conn (worker-state/get-datascript-conn repo)
-         data (ldb/get-view-data repo @conn view-id)]
+         data (db-view/get-view-data repo @conn view-id)]
      (ldb/write-transit-str data)))
 
   (dangerousRemoveAllDbs