refactor: performance for filters

src/main/frontend/components/block.cljs

@@ -48,8 +48,7 @@
             [frontend.commands :as commands]
             [lambdaisland.glogi :as log]
             [frontend.context.i18n :as i18n]
-            [frontend.template :as template]
-            [frontend.filtering :as filtering]))
+            [frontend.template :as template]))
 
 ;; TODO: remove rum/with-context because it'll make reactive queries not working
 
@@ -1571,10 +1570,10 @@
      (when ref?
        (let [children (-> (db/get-block-immediate-children repo uuid)
                           db/sort-by-pos)
-             filtered-children (filtering/filter-blocks children (:filter-state config))]
-         (when (seq filtered-children)
+             children (block-handler/filter-blocks repo children (:filters config) false)]
+         (when (seq children)
            [:div.ref-children.ml-12
-            (blocks-container filtered-children (assoc config
+            (blocks-container children (assoc config
                                               :breadcrumb-show? false
                                               :ref? true))])))
 

src/main/frontend/components/reference.cljs

@@ -14,7 +14,8 @@
             [frontend.config :as config]
             [frontend.components.svg :as svg]
             [frontend.handler.page :as page-handler]
-            [frontend.filtering :as filtering]))
+            [frontend.handler.block :as block-handler]
+            [medley.core :as medley]))
 
 (rum/defc filter-dialog-inner < rum/reactive
   [close-fn references page-name]
@@ -54,12 +55,13 @@
           block-id (and block? (uuid page-name))
           page-name (string/lower-case page-name)
           journal? (date/valid-journal-title? (string/capitalize page-name))
+          repo (state/get-current-repo)
           ref-blocks (cond
                        priority?
-                       (db/get-blocks-by-priority (state/get-current-repo) page-name)
+                       (db/get-blocks-by-priority repo page-name)
 
                        marker?
-                       (db/get-marker-blocks (state/get-current-repo) page-name)
+                       (db/get-marker-blocks repo page-name)
                        block-id
                        (db/get-block-referenced-blocks block-id)
                        :else
@@ -67,10 +69,14 @@
           scheduled-or-deadlines (if journal?
                                    (db/get-date-scheduled-or-deadlines (string/capitalize page-name))
                                    nil)
-          references (remove #{page-name} (distinct (flatten (map filtering/get-nested-block-references (flatten (map val ref-blocks))))))
+          references (db/get-page-linked-refs-refed-pages repo page-name)
           filter-state (rum/react (page-handler/get-filter page-name))
-          filtered-ref-blocks (map #(filtering/filter-ref-block % filter-state) ref-blocks)
-          n-ref (count ref-blocks)]
+          included (filter (fn [[x v]]) filter-state)
+          filters (when (seq filter-state)
+                    (->> (group-by second filter-state)
+                         (medley/map-vals #(map first %))))
+          filtered-ref-blocks (block-handler/filter-blocks repo ref-blocks filters true)
+          n-ref (count filtered-ref-blocks)]
       (when (or (> n-ref 0)
                 (seq scheduled-or-deadlines))
         [:div.references.mt-6.flex-1.flex-row
@@ -111,7 +117,7 @@
                                               :breadcrumb-show? true
                                               :group-by-page? true
                                               :editor-box editor/box
-                                              :filter-state filter-state}
+                                              :filters filters}
                                              {})]
               (content/content page-name
                                {:hiccup ref-hiccup}))])]]))))

src/main/frontend/db.cljs

@@ -46,7 +46,7 @@
   get-date-scheduled-or-deadlines get-db-type get-empty-pages get-file get-file-after-blocks get-file-after-blocks-meta
   get-file-blocks get-file-contents get-file-last-modified-at get-file-no-sub get-file-page get-file-page-id
   get-file-pages get-files get-files-blocks get-files-full get-files-that-referenced-page get-journals-length
-  get-latest-journals get-marker-blocks get-matched-blocks get-page get-page-alias get-page-alias-names get-page-blocks
+  get-latest-journals get-marker-blocks get-matched-blocks get-page get-page-alias get-page-alias-names get-page-blocks get-page-linked-refs-refed-pages
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-name get-page-properties
   get-page-properties-content get-page-referenced-blocks get-page-referenced-pages get-page-unlinked-references
   get-pages get-pages-relation get-pages-that-mentioned-page get-public-pages get-tag-pages

src/main/frontend/db/model.cljs

@@ -836,6 +836,24 @@
                          db-utils/seq-flatten)]
       (mapv (fn [page] [page (get-page-alias repo page)]) ref-pages))))
 
+(defn get-page-linked-refs-refed-pages
+  [repo page]
+  (when-let [conn (conn/get-conn repo)]
+    (->
+     (d/q
+       '[:find [?ref-page ...]
+         :in $ % ?page
+         :where
+         [?p :page/name ?page]
+         [?b :block/path-ref-pages ?p]
+         [?b :block/ref-pages ?other-p]
+         [(not= ?p ?other-p)]
+         [?other-p :page/name ?ref-page]]
+       conn
+       rules
+       page)
+     (distinct))))
+
 ;; Ignore files with empty blocks for now
 (defn get-empty-pages
   [repo]

src/main/frontend/filtering.cljs

@@ -1,43 +0,0 @@
-(ns frontend.filtering
-  (:require [frontend.util :as util]
-            [frontend.state :as state]
-            [frontend.db :as db]))
-
-(defn get-block-references
-  [block]
-  (map #(if (= (first %) "Tag") (second %) (second (:url (second %))))
-       (filter #(and (some (partial = (first %)) ["Tag" "Link"])
-                     (or (= (first %) "Tag")
-                         (= (first (:url (second %))) "Search")))
-               (:block/title block))))
-
-(defn get-nested-block-references
-  [block]
-  (let [repo (state/get-current-repo)
-        children (-> (db/get-block-immediate-children repo (:block/uuid block))
-                     db/sort-by-pos)
-        child-refs (map get-nested-block-references children)
-        block-refs (get-block-references block)]
-    (flatten (concat child-refs block-refs))))
-
-(defn matches-filter
-  [references filter-state]
-  (every? #(= (util/in? (first %) references) (second %)) filter-state))
-
-(defn block-matches-filter
-  [block filter-state]
-  (let [repo (state/get-current-repo)
-        children (-> (db/get-block-immediate-children repo (:block/uuid block))
-                     db/sort-by-pos)
-        block-matches (matches-filter (get-block-references block) filter-state)
-        child-matches (delay (some #(block-matches-filter % filter-state) children))]
-    (or block-matches @child-matches)))
-
-(defn filter-blocks
-  [blocks filter-state]
-  (filter #(block-matches-filter % filter-state) blocks))
-
-(defn filter-ref-block
-  [ref-block filter-state]
-  (let [blocks-filtered (filterv #(block-matches-filter % filter-state) (val ref-block))]
-    (assoc ref-block 1 blocks-filtered)))

src/main/frontend/handler/block.cljs

@@ -6,7 +6,9 @@
             [frontend.format.mldoc :as mldoc]
             [frontend.date :as date]
             [frontend.config :as config]
-            [datascript.core :as d]))
+            [datascript.core :as d]
+            [clojure.set :as set]
+            [medley.core :as medley]))
 
 (defn blocks->vec-tree [col]
   (let [col (map (fn [h] (cond->
@@ -193,3 +195,40 @@
                              :block/pre-block? false})
                           default-option)]
          (conj blocks dummy))))))
+
+(defn filter-blocks
+  [repo ref-blocks filters group-by-page?]
+  (let [ref-pages (->> (if group-by-page?
+                         (mapcat last ref-blocks)
+                         ref-blocks)
+                       (mapcat :block/ref-pages)
+                       (distinct)
+                       (map :db/id)
+                       (db/pull-many repo '[:db/id :page/name]))
+
+        ref-pages (zipmap (map :page/name ref-pages) (map :db/id ref-pages))
+        exclude-ids (->> (map (fn [page] (get ref-pages page)) (get filters false))
+                         (remove nil?)
+                         (set))
+        include-ids (->> (map (fn [page] (get ref-pages page)) (get filters true))
+                         (remove nil?)
+                         (set))]
+    (if (empty? filters)
+      ref-blocks
+      (let [filter-f (fn [ref-blocks]
+                       (cond->> ref-blocks
+                         (seq exclude-ids)
+                         (remove (fn [block]
+                                   (let [ids (set (map :db/id (:block/ref-pages block)))]
+                                     (seq (set/intersection exclude-ids ids)))))
+
+                         (seq include-ids)
+                         (remove (fn [block]
+                                   (let [ids (set (map :db/id (:block/ref-pages block)))]
+                                     (empty? (set/intersection include-ids ids)))))))]
+        (if group-by-page?
+          (->> (map (fn [[p ref-blocks]]
+                      [p (filter-f ref-blocks)]) ref-blocks)
+               (remove #(empty? (second %))))
+          (->> (filter-f ref-blocks)
+               (remove nil?)))))))