refactor: db based filters

Uses properties :logseq.property.linked-references/included-pages
and :logseq.property.linked-references/excluded-pages to store filters
instead of a map.

Fixes LOG-3074

deps/db/src/logseq/db/frontend/property.cljs

@@ -86,6 +86,12 @@
    :logseq.property/order-list-type {:name :logseq.order-list-type
                                      :schema {:type :default
                                               :hide? true}}
+   :logseq.property.linked-references/included-pages {:schema {:type :page
+                                                               :cardinality :many
+                                                               :hide? true}}
+   :logseq.property.linked-references/excluded-pages {:schema {:type :page
+                                                               :cardinality :many
+                                                               :hide? true}}
    :logseq.property.tldraw/page {:name :logseq.tldraw.page
                                  :schema {:type :map
                                           :hide? true}}
@@ -146,8 +152,6 @@
                              :hide? true
                              :view-context :page
                              :public? true}}
-   :logseq.property/filters {:schema {:type :map
-                                      :hide? true}}
    :logseq.property/exclude-from-graph-view {:schema
                                              {:type :checkbox
                                               :hide? true

deps/outliner/src/logseq/outliner/property.cljs

@@ -326,9 +326,12 @@
               fv (first current-val)]
           (if (and (= 1 (count current-val)) (or (= property-value fv) (= property-value (:db/id fv))))
             (remove-block-property! conn (:db/id block) property-id)
-            (ldb/transact! conn
-                           [[:db/retract (:db/id block) property-id property-value]]
-                           {:outliner-op :save-block})))))))
+            (do
+              (prn :debug :tx-data
+                   [[:db/retract (:db/id block) property-id property-value]])
+              (ldb/transact! conn
+                            [[:db/retract (:db/id block) property-id property-value]]
+                            {:outliner-op :save-block}))))))))
 
 (defn collapse-expand-block-property!
   "Notice this works only if the value itself if a block (property type should be either :default or :template)"

src/main/frontend/components/reference.cljs

@@ -1,103 +1,24 @@
 (ns frontend.components.reference
-  (:require [clojure.string :as string]
-            [frontend.config :as config]
-            [frontend.components.block :as block]
+  (:require [frontend.components.block :as block]
             [frontend.components.content :as content]
             [frontend.components.editor :as editor]
+            [frontend.components.reference-filters :as filters]
+            [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
             [frontend.db-mixins :as db-mixins]
+            [frontend.db.async :as db-async]
             [frontend.db.utils :as db-utils]
-            [frontend.db.model :as model-db]
             [frontend.handler.block :as block-handler]
             [frontend.handler.page :as page-handler]
+            [frontend.modules.outliner.tree :as tree]
             [frontend.search :as search]
             [frontend.state :as state]
             [frontend.ui :as ui]
-            [logseq.shui.ui :as shui]
             [frontend.util :as util]
-            [rum.core :as rum]
-            [frontend.modules.outliner.tree :as tree]
-            [frontend.db.async :as db-async]
-            [promesa.core :as p]))
-
-(defn- frequencies-sort
-  [references]
-  (sort-by second #(> %1 %2) references))
-
-(defn filtered-refs
-  [page filters filters-atom filtered-references]
-  [:div.flex.gap-2.flex-wrap.items-center
-   (for [[ref-name ref-count] filtered-references]
-     (when ref-name
-       (let [lc-reference (string/lower-case ref-name)]
-         (ui/button
-           [:span
-            ref-name
-            (when ref-count [:sup " " ref-count])]
-           :on-click (fn [e]
-                       (swap! filters-atom #(if (nil? (get filters lc-reference))
-                                              (assoc % lc-reference (not (.-shiftKey e)))
-                                              (dissoc % lc-reference)))
-                       (page-handler/save-filter! page @filters-atom))
-           :small? true
-           :variant :outline
-           :key ref-name))))])
-
-(rum/defcs filter-dialog-inner < rum/reactive (rum/local "" ::filterSearch)
-  [state page-entity filters-atom *references]
-  (let [filter-search (get state ::filterSearch)
-        references (rum/react *references)
-        filtered-references  (frequencies-sort
-                              (if (= @filter-search "")
-                                references
-                                (search/fuzzy-search references @filter-search :limit 500 :extract-fn first)))
-        filters (rum/react filters-atom)
-        includes (keep (fn [[page include?]]
-                         (let [page' (model-db/get-page-original-name page)]
-                           (when include? [page'])))
-                       filters)
-        excludes (keep (fn [[page include?]]
-                         (let [page' (model-db/get-page-original-name page)]
-                           (when-not include? [page'])))
-                       filters)]
-    [:div.ls-filters.filters
-     [:div.sm:flex.sm:items-start
-      [:div.mx-auto.flex-shrink-0.flex.items-center.justify-center.h-12.w-12.rounded-full.bg-gray-200.text-gray-500.sm:mx-0.sm:h-10.sm:w-10
-       (ui/icon "filter" {:size 20})]
-      [:div.mt-3.text-center.sm:mt-0.sm:ml-4.sm:text-left.pb-2
-       [:h3#modal-headline.text-lg.leading-6.font-medium (t :linked-references/filter-heading)]
-       [:span.text-xs
-        (t :linked-references/filter-directions)]]]
-     (when (seq filters)
-       [:div.cp__filters.mb-4.ml-2
-        (when (seq includes)
-          [:div.flex.flex-row.flex-wrap.center-items
-           [:div.mr-1.font-medium.py-1 (t :linked-references/filter-includes)]
-           (filtered-refs page-entity filters filters-atom includes)])
-        (when (seq excludes)
-          [:div.flex.flex-row.flex-wrap
-           [:div.mr-1.font-medium.py-1 (t :linked-references/filter-excludes)]
-
-           (filtered-refs page-entity filters filters-atom excludes)])])
-     [:div.cp__filters-input-panel.flex.focus-within:bg-gray-03
-      (ui/icon "search")
-      [:input.cp__filters-input.w-full.bg-transparent
-       {:placeholder (t :linked-references/filter-search)
-        :autofocus true
-        :on-change (fn [e]
-                     (reset! filter-search (util/evalue e)))}]]
-     (let [all-filters (set (keys filters))
-           refs (remove (fn [[page _]] (all-filters (util/page-name-sanity-lc page)))
-                        filtered-references)]
-       (when (seq refs)
-         [:div.mt-4
-          (filtered-refs page-entity filters filters-atom refs)]))]))
-
-(defn filter-dialog
-  [page-entity filters-atom *references]
-  (fn []
-    (filter-dialog-inner page-entity filters-atom *references)))
+            [logseq.shui.ui :as shui]
+            [promesa.core :as p]
+            [rum.core :as rum]))
 
 (rum/defc block-linked-references < rum/reactive db-mixins/query
   {:init (fn [state]
@@ -138,13 +59,15 @@
      (content/content page-name {:hiccup ref-hiccup}))])
 
 (rum/defc references-cp
-  [page-entity page-name filters filters-atom filter-state total filter-n filtered-ref-blocks *ref-pages]
-  (let [threshold (state/get-linked-references-collapsed-threshold)
+  [page-entity page-name *filters total filter-n filtered-ref-blocks *ref-pages]
+  (let [filters @*filters
+        threshold (state/get-linked-references-collapsed-threshold)
         default-collapsed? (>= total threshold)
         *collapsed? (atom nil)]
     (ui/foldable
      [:div.flex.flex-row.flex-1.justify-between.items-center
-      [:h2.font-medium (t :linked-references/reference-count (if (seq filters) filter-n nil) total)]
+      [:h2.font-medium (t :linked-references/reference-count (when (or (seq (:included filters))
+                                                                       (seq (:excluded filters))) filter-n) total)]
       [:a.filter.fade-link
        {:title (t :linked-references/filter-heading)
         :on-mouse-over (fn [_e]
@@ -153,14 +76,19 @@
                            (reset! @*collapsed? false)))
         :on-pointer-down (fn [e]
                            (util/stop-propagation e)
-                           (shui/dialog-open!
-                             (filter-dialog page-entity filters-atom *ref-pages)))}
+                           (shui/popup-show! (.-target e)
+                            (fn []
+                              [:div.p-4
+                               (filters/filter-dialog page-entity *filters *ref-pages)])
+                            {:align "end"}))}
        (ui/icon "filter" {:class (cond
-                                   (empty? filter-state)
+                                   (and (empty? (:included filters)) (empty? (:excluded filters)))
                                    "opacity-60 hover:opacity-100"
-                                   (every? true? (vals filter-state))
+
+                                   (and (seq (:included filters)) (empty? (:excluded filters)))
                                    "text-success"
-                                   (every? false? (vals filter-state))
+
+                                   (and (empty? (:included filters)) (seq (:excluded filters)))
                                    "text-error"
                                    :else
                                    "text-warning")
@@ -188,41 +116,30 @@
             (concat children (rest blocks))
             (conj result fb))))))))
 
-(rum/defc sub-page-properties-changed < rum/static
-  [page-entity v filters-atom]
-  (rum/use-effect!
-   (fn []
-     (reset! filters-atom
-             (page-handler/get-filters page-entity)))
-   [page-entity v filters-atom])
-  [:<>])
-
 (rum/defcs references* < rum/reactive db-mixins/query
   (rum/local nil ::ref-pages)
   {:init (fn [state]
-           (let [page (first (:rum/args state))
-                 filters (when page (atom nil))]
-             (when page (db-async/<get-block-refs (state/get-current-repo) (:db/id page)))
-             (assoc state ::filters filters)))}
+           (let [page (first (:rum/args state))]
+             (when page (db-async/<get-block-refs (state/get-current-repo) (:db/id page))))
+           (assoc state ::filters (atom nil)))}
   [state page-entity]
   (when page-entity
-    (let [repo (state/get-current-repo)]
+    (let [repo (state/get-current-repo)
+          *filters (::filters state)]
       (when page-entity
         (when-not (state/sub-async-query-loading (str (:db/id page-entity) "-refs"))
-          (let [page-name (:block/name page-entity)
-                page-props-v (state/sub-page-properties-changed page-name)
+          (let [page-entity (db/sub-block (:db/id page-entity))
+                page-name (:block/name page-entity)
                 *ref-pages (::ref-pages state)
-                filters-atom (get state ::filters)
-                filter-state (rum/react filters-atom)
+                filters (page-handler/get-filters page-entity)
+                _ (when-not (= filters @*filters)
+                    (reset! *filters filters))
                 page-id (:db/id page-entity)
                 ref-blocks (db/get-page-referenced-blocks page-id)
                 aliases (db/page-alias-set repo page-id)
                 aliases-exclude-self (set (remove #{page-id} aliases))
                 top-level-blocks (filter (fn [b] (some aliases (set (map :db/id (:block/refs b))))) ref-blocks)
                 top-level-blocks-ids (set (map :db/id top-level-blocks))
-                filters (when (seq filter-state)
-                          (-> (group-by second filter-state)
-                              (update-vals #(map first %))))
                 filtered-ref-blocks (->> (block-handler/filter-blocks ref-blocks filters)
                                          (block-handler/get-filtered-ref-blocks-with-parents ref-blocks))
                 total (count top-level-blocks)
@@ -251,11 +168,10 @@
                            (map :block/original-name)
                            frequencies)]
             (reset! *ref-pages ref-pages)
-            (when (or (seq filter-state) (> filter-n 0))
+            (when (or (seq (:included filters)) (seq (:excluded filters)) (> filter-n 0))
               [:div.references.page-linked.flex-1.flex-row
-               (sub-page-properties-changed page-entity page-props-v filters-atom)
                [:div.content.pt-6
-                (references-cp page-entity page-name filters filters-atom filter-state total filter-n filtered-ref-blocks' *ref-pages)]])))))))
+                (references-cp page-entity page-name *filters total filter-n filtered-ref-blocks' *ref-pages)]])))))))
 
 (rum/defc references
   [page-entity]

src/main/frontend/components/reference_filters.cljs

@@ -0,0 +1,96 @@
+(ns frontend.components.reference-filters
+  "References filters"
+  (:require [clojure.string :as string]
+            [frontend.context.i18n :refer [t]]
+            [frontend.handler.page :as page-handler]
+            [frontend.search :as search]
+            [frontend.ui :as ui]
+            [frontend.util :as util]
+            [rum.core :as rum]
+            [frontend.config :as config]
+            [frontend.state :as state]
+            [frontend.db :as db]
+            [datascript.impl.entity :as de]))
+
+(defn- frequencies-sort
+  [references]
+  (sort-by second #(> %1 %2) references))
+
+(defn filtered-refs
+  [page filters filtered-references*]
+  [:div.flex.gap-2.flex-wrap.items-center
+   (let [filtered-references (if (de/entity? (first filtered-references*))
+                               (map (fn [e] [(:block/original-name e)]) filtered-references*)
+                               filtered-references*)] <
+        (for [[ref-name ref-count] filtered-references]
+          (when ref-name
+            (let [lc-reference (string/lower-case ref-name)]
+              (ui/button
+               [:span
+                ref-name
+                (when ref-count [:sup " " ref-count])]
+               :on-click (fn [e]
+                           (let [db-based? (config/db-based-graph? (state/get-current-repo))
+                                 includes (set (map :block/name (:included filters)))
+                                 excludes (set (map :block/name (:excluded filters)))
+                                 included? (includes lc-reference)
+                                 not-in-filters? (and (not included?) (not (excludes lc-reference)))
+                                 shift? (.-shiftKey e)]
+                             (if db-based?
+                               (page-handler/db-based-save-filter! page (:db/id (db/get-page lc-reference))
+                                                                   {:add? not-in-filters?
+                                                                    :include? (if not-in-filters? (not shift?) included?)})
+                               (let [filters-m (->> (concat (map #(vector % true) includes) (map #(vector % false) excludes))
+                                                    (into {}))
+                                     filters' (if not-in-filters?
+                                                (assoc filters-m lc-reference (not shift?))
+                                                (dissoc filters-m lc-reference))]
+                                 (page-handler/file-based-save-filter! page filters')))))
+               :small? true
+               :variant :outline
+               :key ref-name)))))])
+
+(rum/defcs filter-dialog < rum/reactive (rum/local "" ::filterSearch)
+  [state page-entity *filters *references]
+  (let [filters (rum/react *filters)
+        filter-search (get state ::filterSearch)
+        references (rum/react *references)
+        filtered-references  (frequencies-sort
+                              (if (= @filter-search "")
+                                references
+                                (search/fuzzy-search references @filter-search :limit 500 :extract-fn first)))
+        {:keys [included excluded]} filters]
+    [:div.ls-filters.filters
+     [:div.sm:flex.sm:items-start
+      [:div.mx-auto.flex-shrink-0.flex.items-center.justify-center.h-12.w-12.rounded-full.bg-gray-200.text-gray-500.sm:mx-0.sm:h-10.sm:w-10
+       (ui/icon "filter" {:size 20})]
+      [:div.mt-3.text-center.sm:mt-0.sm:ml-4.sm:text-left.pb-2
+       [:h3#modal-headline.text-lg.leading-6.font-medium (t :linked-references/filter-heading)]
+       [:span.text-xs
+        (t :linked-references/filter-directions)]]]
+     (when (or (seq included) (seq excluded))
+       [:div.cp__filters.mb-4.ml-2
+        (when (seq included)
+          [:div.flex.flex-row.flex-wrap.center-items
+           [:div.mr-1.font-medium.py-1 (t :linked-references/filter-includes)]
+           (filtered-refs page-entity filters included)])
+        (when (seq excluded)
+          [:div.flex.flex-row.flex-wrap
+           [:div.mr-1.font-medium.py-1 (t :linked-references/filter-excludes)]
+
+           (filtered-refs page-entity filters excluded)])])
+     [:div.cp__filters-input-panel.flex.focus-within:bg-gray-03
+      (ui/icon "search")
+      [:input.cp__filters-input.w-full.bg-transparent
+       {:placeholder (t :linked-references/filter-search)
+        :autofocus true
+        :on-change (fn [e]
+                     (reset! filter-search (util/evalue e)))}]]
+     (let [all-filters (set
+                        (concat (map :block/name included)
+                                (map :block/name excluded)))
+           refs (remove (fn [[page _]] (all-filters (util/page-name-sanity-lc page)))
+                        filtered-references)]
+       (when (seq refs)
+         [:div.mt-4
+          (filtered-refs page-entity filters refs)]))]))

src/main/frontend/db/model.cljs

@@ -235,16 +235,16 @@ independent of format as format specific heading characters are stripped"
 (def sort-by-order ldb/sort-by-order)
 
 (defn sub-block
-  [id]
+  [id & {:keys [ref?]}]
   (when-let [repo (state/get-current-repo)]
-    (->
-     (react/q repo [:frontend.worker.react/block id]
-              {:query-fn (fn [_]
-                           (let [e (db-utils/entity id)]
-                             [e (:block/tx-id e)]))}
-              nil)
-     react
-     first)))
+    (let [ref (react/q repo [:frontend.worker.react/block id]
+                       {:query-fn (fn [_]
+                                    (let [e (db-utils/entity id)]
+                                      [e (:block/tx-id e)]))}
+                       nil)]
+      (if ref?
+        ref
+        (-> ref react first)))))
 
 (defn sort-by-order-recursive
   [form]
@@ -505,12 +505,6 @@ independent of format as format specific heading characters are stripped"
                (:block/name page-entity)
                page-name)))))))
 
-(defn get-page-original-name
-  [page-name]
-  (when (string? page-name)
-    (let [page (ldb/get-page (conn/get-db) page-name)]
-      (:block/original-name page))))
-
 (defn get-journals-length
   []
   (let [today (date-time-util/date->int (js/Date.))]

src/main/frontend/handler/block.cljs

@@ -87,10 +87,8 @@
   [ref-blocks filters]
   (if (empty? filters)
     ref-blocks
-    (let [exclude-ids (->> (keep (fn [page] (:db/id (db/get-page page))) (get filters false))
-                           (set))
-          include-ids (->> (keep (fn [page] (:db/id (db/get-page page))) (get filters true))
-                           (set))]
+    (let [exclude-ids (set (map :db/id (:excluded filters)))
+          include-ids (set (map :db/id (:included filters)))]
       (cond->> ref-blocks
         (seq exclude-ids)
         (remove (fn [block]

src/main/frontend/handler/editor.cljs

@@ -268,17 +268,10 @@
                       ;; :block/uuid might be changed when backspace/delete
                       ;; a block that has been refed
                       (assoc :block/uuid (:block/uuid block)))
-           opts' (assoc opts :outliner-op :save-block)
-           original-block (db/entity (:db/id block))
-           original-props (:block/properties original-block)]
+           opts' (assoc opts :outliner-op :save-block)]
        (ui-outliner-tx/transact!
                    opts'
-                   (outliner-save-block! block')
-                   ;; page properties changed
-                   (when-let [page-name (and (:block/pre-block? block')
-                                             (not= original-props (:block/properties block'))
-                                             (some-> (:block/page block') :db/id (db-utils/pull) :block/name))]
-                     (state/set-page-properties-changed! page-name)))))))
+                   (outliner-save-block! block'))))))
 
 ;; id: block dom id, "ls-block-counter-uuid"
 (defn- another-block-with-same-id-exists?

src/main/frontend/handler/page.cljs

@@ -15,6 +15,7 @@
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.notification :as notification]
             [frontend.handler.property :as property-handler]
+            [frontend.handler.db-based.property :as db-property-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.file-based.nfs :as nfs-handler]
             [frontend.handler.graph :as graph-handler]
@@ -258,18 +259,40 @@
 
 (defn get-filters
   [page]
-  (let [k (pu/get-pid :logseq.property/filters)]
-    (if (config/db-based-graph? (state/get-current-repo))
-      (get page k)
-      (let [properties (:block/properties page)
-            properties-str (or (get properties k) "{}")]
-        (try (reader/read-string properties-str)
-             (catch :default e
-               (log/error :syntax/filters e)))))))
-
-(defn save-filter!
+  (if (config/db-based-graph? (state/get-current-repo))
+    (let [included-pages (:logseq.property.linked-references/included-pages page)
+          excluded-pages (:logseq.property.linked-references/excluded-pages page)]
+      {:included included-pages
+       :excluded excluded-pages})
+    (let [k :filters
+          properties (:block/properties page)
+          properties-str (or (get properties k) "{}")]
+      (try (let [result (reader/read-string properties-str)]
+             (when (seq result)
+               (let [excluded-pages (->> (filter #(false? (second %)) result)
+                                         (keep first)
+                                         (keep db/get-page))
+                     included-pages (->> (filter #(true? (second %)) result)
+                                         (keep first)
+                                         (keep db/get-page))]
+                 {:included included-pages
+                  :excluded excluded-pages})))
+           (catch :default e
+             (log/error :syntax/filters e))))))
+
+(defn file-based-save-filter!
   [page filter-state]
-  (property-handler/add-page-property! page (pu/get-pid :logseq.property/filters) filter-state))
+  (property-handler/add-page-property! page :filters filter-state))
+
+(defn db-based-save-filter!
+  [page filter-page-id {:keys [include? add?]}]
+  (let [repo (state/get-current-repo)
+        property-id (if include?
+                      :logseq.property.linked-references/included-pages
+                      :logseq.property.linked-references/excluded-pages)]
+    (if add?
+      (property-handler/set-block-property! repo (:db/id page) property-id filter-page-id)
+      (db-property-handler/delete-property-value! (:db/id page) property-id filter-page-id))))
 
 ;; Editor
 (defn page-not-exists-handler

src/main/frontend/state.cljs

@@ -151,7 +151,6 @@
       :editor/code-block-context             {}
       :editor/latest-shortcut                (atom nil)
 
-      :db/properties-changed-pages           {}
       :history/paused?                       (atom false)
       :editor/cursor-range                   (atom nil)
       :editor/container-id                   (atom nil)
@@ -2376,16 +2375,6 @@ Similar to re-frame subscriptions"
   (js/setTimeout #(async/go
                     (>! (get-handbook-route-chan) k))))
 
-(defn set-page-properties-changed!
-  [page-name]
-  (when-not (string/blank? page-name)
-    (update-state! [:db/properties-changed-pages page-name] #(inc %))))
-
-(defn sub-page-properties-changed
-  [page-name]
-  (when-not (string/blank? page-name)
-    (sub [:db/properties-changed-pages page-name])))
-
 (defn update-favorites-updated!
   []
   (update-state! :favorites/updated? inc))