Move all search related impl to worker

This commit also introduced a new ns `frontend.db.async` for
async queries.

shadow-cljs.edn

@@ -118,4 +118,3 @@
                :devtools         {:before-load frontend.core/stop
                                   :after-load  frontend.core/start
                                   :preloads    [devtools.preload]}}}}
-

src/main/frontend/components/cmdk.cljs

@@ -754,13 +754,14 @@
                     (on-blur input)))
        :on-composition-end (fn [e] (handle-input-change state e))
        :on-key-down (fn [e]
-                      (let [value (.-value @input-ref)
-                            last-char (last value)
-                            backspace? (= (util/ekey e) "Backspace")
-                            filter-group (:group @(::filter state))
-                            slash? (= (util/ekey e) "/")
-                            namespace-page-matched? (when (and slash? (contains? #{:pages :whiteboards} filter-group))
-                                                      (some #(string/includes? % "/") (search/page-search (str value "/"))))]
+                      (p/let [value (.-value @input-ref)
+                              last-char (last value)
+                              backspace? (= (util/ekey e) "Backspace")
+                              filter-group (:group @(::filter state))
+                              slash? (= (util/ekey e) "/")
+                              namespace-pages (when (and slash? (contains? #{:pages :whiteboards} filter-group))
+                                                (search/page-search (str value "/")))
+                              namespace-page-matched? (some #(string/includes? % "/") namespace-pages)]
                         (when (and filter-group
                                    (or (and slash? (not namespace-page-matched?))
                                        (and backspace? (= last-char "/"))

src/main/frontend/components/content.cljs

@@ -31,7 +31,8 @@
             [frontend.db.rtc.debug-ui :as rtc-debug-ui]
             [cljs.core.async :as async]
             [cljs.pprint :as pp]
-            [cljs-time.coerce :as tc]))
+            [cljs-time.coerce :as tc]
+            [promesa.core :as p]))
 
 ;; TODO i18n support
 
@@ -156,15 +157,16 @@
                      :on-click (fn []
                                  (let [title (string/trim @input)]
                                    (when (not (string/blank? title))
-                                     (if (page-handler/template-exists? title)
-                                       (notification/show!
-                                        [:p (t :context-menu/template-exists-warning)]
-                                        :error)
-                                       (do
-                                         (property-handler/set-block-property! repo block-id :template title)
-                                         (when (false? template-including-parent?)
-                                           (property-handler/set-block-property! repo block-id :template-including-parent false))
-                                         (state/hide-custom-context-menu!)))))))]
+                                     (p/let [exists? (page-handler/<template-exists? title)]
+                                       (if exists?
+                                         (notification/show!
+                                          [:p (t :context-menu/template-exists-warning)]
+                                          :error)
+                                         (do
+                                           (property-handler/set-block-property! repo block-id :template title)
+                                           (when (false? template-including-parent?)
+                                             (property-handler/set-block-property! repo block-id :template-including-parent false))
+                                           (state/hide-custom-context-menu!))))))))]
          [:hr.menu-separator]])
       (ui/menu-link
        {:key "Make a Template"

src/main/frontend/components/editor.cljs

@@ -121,12 +121,73 @@
                                                  :other-attrs {:block/link (:db/id (db/entity [:block/name page-name]))}}))))
     (page-handler/on-chosen-handler input id q pos format)))
 
-(rum/defcs page-search < rum/reactive
+(rum/defc page-search-aux
+  [id format embed? db-tag? create-page? q current-pos edit-content input pos]
+  (let [[matched-pages set-matched-pages!] (rum/use-state nil)]
+    (rum/use-effect! (fn []
+                       (when-not (string/blank? q)
+                         (p/let [result (editor-handler/<get-matched-pages q)]
+                           (set-matched-pages! result))))
+                     [q])
+    (let [matched-pages (cond
+                          (contains? (set (map util/page-name-sanity-lc matched-pages))
+                                     (util/page-name-sanity-lc (string/trim q)))  ;; if there's a page name fully matched
+                          (sort-by (fn [m]
+                                     [(count m) m])
+                                   matched-pages)
+
+                          (string/blank? q)
+                          nil
+
+                          (empty? matched-pages)
+                          (when-not (db/page-exists? q)
+                            (if db-tag?
+                              (concat [(str (t :new-page) " " q)
+                                       (str (t :new-class) " " q)]
+                                      matched-pages)
+                              (cons q matched-pages)))
+
+                                ;; reorder, shortest and starts-with first.
+                          :else
+                          (let [matched-pages (remove nil? matched-pages)
+                                matched-pages (sort-by
+                                               (fn [m]
+                                                 [(not (gstring/caseInsensitiveStartsWith m q)) (count m) m])
+                                               matched-pages)]
+                            (if (gstring/caseInsensitiveStartsWith (first matched-pages) q)
+                              (cons (first matched-pages)
+                                    (cons q (rest matched-pages)))
+                              (cons q matched-pages))))]
+      [:div
+       (when (and db-tag?
+                        ;; Don't display in heading
+                  (not (some->> edit-content (re-find #"^\s*#"))))
+         [:div.flex.flex-row.items-center.px-4.py-1.text-sm.opacity-70.gap-2
+          "Turn this block into a page:"
+          (ui/toggle create-page?
+                     (fn [_e]
+                       (swap! (:editor/create-page? @state/state) not))
+                     true)])
+       (ui/auto-complete
+        matched-pages
+        {:on-chosen   (page-on-chosen-handler embed? input id q pos format)
+         :on-enter    (fn []
+                        (page-handler/page-not-exists-handler input id q current-pos))
+         :item-render (fn [page-name _chosen?]
+                        [:div.flex
+                         (when (db-model/whiteboard-page? page-name) [:span.mr-1 (ui/icon "whiteboard" {:extension? true})])
+                         (search-handler/highlight-exact-query page-name q)])
+         :empty-placeholder [:div.text-gray-500.text-sm.px-4.py-2 (if db-tag?
+                                                                    "Search for a page or a class"
+                                                                    "Search for a page")]
+         :class       "black"})])))
+
+(rum/defc page-search < rum/reactive
   {:will-unmount (fn [state]
                    (reset! commands/*current-command nil)
                    state)}
   "Page or tag searching popup"
-  [state id format]
+  [id format]
   (let [action (state/sub :editor/action)
         db? (config/db-based-graph? (state/get-current-repo))
         embed? (and db? (= @commands/*current-command "Page embed"))
@@ -145,63 +206,8 @@
                      (gp-util/safe-subs edit-content pos current-pos))
                    (when (> (count edit-content) current-pos)
                      (gp-util/safe-subs edit-content pos current-pos))
-                   "")
-                ;; FIXME: display refed pages recentedly or frequencyly used
-                matched-pages (when-not (string/blank? q)
-                                (editor-handler/get-matched-pages q))
-                matched-pages (cond
-                                (contains? (set (map util/page-name-sanity-lc matched-pages))
-                                           (util/page-name-sanity-lc (string/trim q)))  ;; if there's a page name fully matched
-                                (sort-by (fn [m]
-                                           [(count m) m])
-                                         matched-pages)
-
-                                (string/blank? q)
-                                nil
-
-                                (empty? matched-pages)
-                                (when-not (db/page-exists? q)
-                                  (if db-tag?
-                                    (concat [(str (t :new-page) " " q)
-                                             (str (t :new-class) " " q)]
-                                            matched-pages)
-                                    (cons q matched-pages)))
-
-                                ;; reorder, shortest and starts-with first.
-                                :else
-                                (let [matched-pages (remove nil? matched-pages)
-                                      matched-pages (sort-by
-                                                     (fn [m]
-                                                       [(not (gstring/caseInsensitiveStartsWith m q)) (count m) m])
-                                                     matched-pages)]
-                                  (if (gstring/caseInsensitiveStartsWith (first matched-pages) q)
-                                    (cons (first matched-pages)
-                                          (cons q (rest matched-pages)))
-                                    (cons q matched-pages))))]
-            [:div
-             (when (and db-tag?
-                        ;; Don't display in heading
-                        (not (some->> edit-content (re-find #"^\s*#"))))
-               [:div.flex.flex-row.items-center.px-4.py-1.text-sm.opacity-70.gap-2
-                "Turn this block into a page:"
-                (ui/toggle create-page?
-                           (fn [_e]
-                             (swap! (:editor/create-page? @state/state) not))
-                           true)])
-             (ui/auto-complete
-              matched-pages
-              {:on-chosen   (page-on-chosen-handler embed? input id q pos format)
-               :on-enter    (fn []
-                              (page-handler/page-not-exists-handler input id q current-pos))
-               :item-render (fn [page-name _chosen?]
-                              [:div.flex
-                               (when (db-model/whiteboard-page? page-name) [:span.mr-1 (ui/icon "whiteboard" {:extension? true})])
-                               (search-handler/highlight-exact-query page-name q)])
-               :empty-placeholder [:div.text-gray-500.text-sm.px-4.py-2 (if db-tag?
-                                                                          "Search for a page or a class"
-                                                                          "Search for a page")]
-               :class       "black"})]))))))
-
+                   "")]
+            (page-search-aux id format embed? db-tag? create-page? q current-pos edit-content input pos)))))))
 
 (defn- search-blocks!
   [state result]
@@ -231,6 +237,7 @@
           (state/clear-edit!))))
     (editor-handler/block-on-chosen-handler id q format selected-text)))
 
+;; TODO: use rum/use-effect instead
 (rum/defcs block-search-auto-complete < rum/reactive
   {:init (fn [state]
            (let [result (atom nil)]
@@ -283,6 +290,22 @@
       (when input
         (block-search-auto-complete edit-block input id q format selected-text)))))
 
+(rum/defc template-search-aux
+  [id q]
+  (let [[matched-templates set-matched-templates!] (rum/use-state nil)]
+    (rum/use-effect! (fn []
+                       (p/let [result (editor-handler/<get-matched-templates q)]
+                         (set-matched-templates! result)))
+                     [q])
+    (ui/auto-complete
+     matched-templates
+     {:on-chosen   (editor-handler/template-on-chosen-handler id)
+      :on-enter    (fn [_state] (state/clear-editor-action!))
+      :empty-placeholder [:div.text-gray-500.px-4.py-2.text-sm "Search for a template"]
+      :item-render (fn [[template _block-db-id]]
+                     template)
+      :class       "black"})))
+
 (rum/defc template-search < rum/reactive
   [id _format]
   (let [pos (state/get-editor-last-pos)
@@ -293,37 +316,32 @@
             q (or
                (when (>= (count edit-content) current-pos)
                  (subs edit-content pos current-pos))
-               "")
-            matched-templates (editor-handler/get-matched-templates q)
-            non-exist-handler (fn [_state]
-                                (state/clear-editor-action!))]
-        (ui/auto-complete
-         matched-templates
-         {:on-chosen   (editor-handler/template-on-chosen-handler id)
-          :on-enter    non-exist-handler
-          :empty-placeholder [:div.text-gray-500.px-4.py-2.text-sm "Search for a template"]
-          :item-render (fn [[template _block-db-id]]
-                         template)
-          :class       "black"})))))
+               "")]
+        (template-search-aux id q)))))
 
-(rum/defc property-search < rum/reactive
+(rum/defc property-search
   [id]
-  (let [input (gdom/getElement id)]
+  (let [input (gdom/getElement id)
+        [matched-properties set-matched-properties!] (rum/use-state nil)]
     (when input
       (let [q (or (:searching-property (editor-handler/get-searching-property input))
-                  "")
-            matched-properties (editor-handler/get-matched-properties q)
-            q-property (string/replace (string/lower-case q) #"\s+" "-")
-            non-exist-handler (fn [_state]
-                                ((editor-handler/property-on-chosen-handler id q-property) nil))]
-        (ui/auto-complete
-         matched-properties
-         {:on-chosen (editor-handler/property-on-chosen-handler id q-property)
-          :on-enter non-exist-handler
-          :empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property: " q-property)]
-          :header [:div.px-4.py-2.text-sm.font-medium "Matched properties: "]
-          :item-render (fn [property] property)
-          :class       "black"})))))
+                  "")]
+        (rum/use-effect!
+         (fn []
+           (p/let [matched-properties (editor-handler/<get-matched-properties q)]
+             (set-matched-properties! matched-properties)))
+         [q])
+        (let [q-property (string/replace (string/lower-case q) #"\s+" "-")
+              non-exist-handler (fn [_state]
+                                  ((editor-handler/property-on-chosen-handler id q-property) nil))]
+          (ui/auto-complete
+           matched-properties
+           {:on-chosen (editor-handler/property-on-chosen-handler id q-property)
+            :on-enter non-exist-handler
+            :empty-placeholder [:div.px-4.py-2.text-sm (str "Create a new property: " q-property)]
+            :header [:div.px-4.py-2.text-sm.font-medium "Matched properties: "]
+            :item-render (fn [property] property)
+            :class       "black"}))))))
 
 (rum/defc property-value-search < rum/reactive
   [id]

src/main/frontend/components/property.cljs

@@ -29,7 +29,8 @@
             [frontend.components.dnd :as dnd]
             [dommy.core :as dom]
             [frontend.components.property.closed-value :as closed-value]
-            [frontend.components.property.util :as components-pu]))
+            [frontend.components.property.util :as components-pu]
+            [promesa.core :as p]))
 
 (def icon closed-value/icon)
 
@@ -345,6 +346,27 @@
         (do (notification/show! "This is an invalid property name. A property name cannot start with page reference characters '#' or '[['." :error)
             (pv/exit-edit-property))))))
 
+(rum/defc property-select
+  [exclude-properties on-chosen input-opts]
+  (let [[properties set-properties!] (rum/use-state nil)]
+    (rum/use-effect!
+     (fn []
+       (p/let [properties (search/get-all-properties)]
+         (set-properties! (remove exclude-properties properties))))
+     [])
+    [:div.ls-property-add.flex.flex-row.items-center
+    [:span.bullet-container.cursor [:span.bullet]]
+    [:div.ls-property-key {:style {:padding-left 6
+                                   :height "1.5em"}} ; TODO: ugly
+     (select/select {:items (map (fn [x] {:value x}) properties)
+                     :dropdown? true
+                     :close-modal? false
+                     :show-new-when-not-exact-match? true
+                     :exact-match-exclude-items exclude-properties
+                     :input-default-placeholder "Add property"
+                     :on-chosen on-chosen
+                     :input-opts input-opts})]]))
+
 (rum/defcs property-input < rum/reactive
   (rum/local false ::show-new-property-config?)
   shortcut/disable-all-shortcuts
@@ -364,9 +386,7 @@
                                    #{}
                                    [:tags :alias])
         exclude-properties* (set/union entity-properties existing-tag-alias)
-        exclude-properties (set/union exclude-properties* (set (map string/lower-case exclude-properties*)))
-        properties (->> (search/get-all-properties)
-                        (remove exclude-properties))]
+        exclude-properties (set/union exclude-properties* (set (map string/lower-case exclude-properties*)))]
     [:div.ls-property-input.flex.flex-1.flex-row.items-center.flex-wrap.gap-1
      (if in-block-container? {:style {:padding-left 22}} {})
      (if @*property-key
@@ -395,26 +415,17 @@
                                "origin-top-right.absolute.left-0.rounded-md.shadow-lg.mt-2")})
                (pv/property-value entity property @*property-value (assoc opts :editing? true))))]])
 
-       [:div.ls-property-add.flex.flex-row.items-center
-        [:span.bullet-container.cursor [:span.bullet]]
-        [:div.ls-property-key {:style {:padding-left 6
-                                       :height "1.5em"}} ; TODO: ugly
-         (select/select {:items (map (fn [x] {:value x}) properties)
-                         :dropdown? true
-                         :close-modal? false
-                         :show-new-when-not-exact-match? true
-                         :exact-match-exclude-items exclude-properties
-                         :input-default-placeholder "Add property"
-                         :on-chosen (fn [{:keys [value]}]
-                                      (reset! *property-key value)
-                                      (add-property-from-dropdown entity value (assoc opts :*show-new-property-config? *show-new-property-config?)))
-                         :input-opts {:on-blur (fn [] (pv/exit-edit-property))
-                                      :on-key-down
-                                      (fn [e]
-                                        (case (util/ekey e)
-                                          "Escape"
-                                          (pv/exit-edit-property)
-                                          nil))}})]])]))
+       (let [on-chosen (fn [{:keys [value]}]
+                         (reset! *property-key value)
+                         (add-property-from-dropdown entity value (assoc opts :*show-new-property-config? *show-new-property-config?)))
+             input-opts {:on-blur (fn [] (pv/exit-edit-property))
+                         :on-key-down
+                         (fn [e]
+                           (case (util/ekey e)
+                             "Escape"
+                             (pv/exit-edit-property)
+                             nil))}]
+         (property-select exclude-properties on-chosen input-opts)))]))
 
 (defonce *last-new-property-input-id (atom nil))
 (rum/defcs new-property < rum/reactive

src/main/frontend/components/query/builder.cljs

@@ -1,9 +1,9 @@
 (ns frontend.components.query.builder
   "DSL query builder."
-  (:require [frontend.config :as config]
-            [frontend.date :as date]
+  (:require [frontend.date :as date]
             [frontend.ui :as ui]
             [frontend.db :as db]
+            [frontend.db.async :as db-async]
             [frontend.db.model :as db-model]
             [frontend.db.query-dsl :as query-dsl]
             [frontend.handler.editor :as editor-handler]
@@ -17,7 +17,8 @@
             [rum.core :as rum]
             [clojure.string :as string]
             [logseq.graph-parser.util :as gp-util]
-            [logseq.graph-parser.util.page-ref :as page-ref]))
+            [logseq.graph-parser.util.page-ref :as page-ref]
+            [promesa.core :as p]))
 
 (rum/defc page-block-selector
   [*find]
@@ -118,6 +119,36 @@
                        (append-tree! tree opts loc clause)
                        (reset! *between-dates {}))))))])
 
+(rum/defc property-select
+  [*mode *property]
+  (let [[properties set-properties!] (rum/use-state nil)]
+    (rum/use-effect!
+     (fn []
+       (p/let [properties (search/get-all-properties)]
+         (set-properties! properties)))
+     [])
+    (select properties
+            (fn [{:keys [value]}]
+              (reset! *mode "property-value")
+              (reset! *property (keyword value))))))
+
+(rum/defc property-value-select
+  [repo *property *find *tree opts loc]
+  (let [[values set-values!] (rum/use-state nil)]
+    (rum/use-effect!
+     (fn []
+       (p/let [result (db-async/<get-property-values repo @*property)]
+         (set-values! result)))
+     [@*property])
+    (let [values (cons "Select all" values)]
+      (select values
+              (fn [{:keys [value]}]
+                (let [x (if (= value "Select all")
+                          [(if (= @*find :page) :page-property :property) @*property]
+                          [(if (= @*find :page) :page-property :property) @*property value])]
+                  (reset! *property nil)
+                  (append-tree! *tree opts loc x)))))))
+
 (defn- query-filter-picker
   [state *find *tree loc clause opts]
   (let [*mode (::mode state)
@@ -140,23 +171,10 @@
                    (append-tree! *tree opts loc [:page-tags value]))))
 
        "property"
-       (let [properties (search/get-all-properties)]
-         (select properties
-                 (fn [{:keys [value]}]
-                   (reset! *mode "property-value")
-                   (reset! *property (keyword value)))))
+       (property-select *mode *property)
 
        "property-value"
-       (let [values (cons "Select all" (if (config/db-based-graph? repo)
-                                         (db-model/get-db-property-values repo @*property)
-                                         (db-model/get-property-values @*property)))]
-         (select values
-                 (fn [{:keys [value]}]
-                   (let [x (if (= value "Select all")
-                             [(if (= @*find :page) :page-property :property) @*property]
-                             [(if (= @*find :page) :page-property :property) @*property value])]
-                     (reset! *property nil)
-                     (append-tree! *tree opts loc x)))))
+       (property-value-select repo *property *find *tree opts loc)
 
        "sample"
        (select (range 1 101)

src/main/frontend/db.cljs

@@ -30,8 +30,8 @@
 
  [frontend.db.model
   delete-blocks get-pre-block
-  delete-files delete-pages-by-files get-all-block-contents get-all-tagged-pages get-single-block-contents
-  get-all-templates get-block-and-children get-block-by-uuid get-block-children sort-by-left
+  delete-files delete-pages-by-files get-all-tagged-pages
+  get-block-and-children get-block-by-uuid get-block-children sort-by-left
   get-block-parent get-block-parents parents-collapsed? get-block-referenced-blocks get-all-referenced-blocks-uuid
   get-block-immediate-children get-block-page
   get-custom-css get-date-scheduled-or-deadlines
@@ -40,7 +40,7 @@
   get-latest-journals get-page get-page-alias get-page-alias-names
   get-page-blocks-count get-page-blocks-no-cache get-page-file get-page-format get-page-properties
   get-page-referenced-blocks get-page-referenced-blocks-full get-page-referenced-pages get-page-unlinked-references
-  get-all-pages get-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages
+  get-all-pages get-pages-relation get-pages-that-mentioned-page get-tag-pages
   journal-page? page-alias-set sub-block
   set-file-last-modified-at! page-empty? page-exists? page-empty-or-dummy? get-alias-source-page
   set-file-content! has-children? get-namespace-pages get-all-namespace-relation get-pages-by-name-partition

src/main/frontend/db/async.cljs

@@ -0,0 +1,100 @@
+(ns frontend.db.async
+  "Async queries"
+  (:require [promesa.core :as p]
+            [frontend.state :as state]
+            [frontend.config :as config]
+            [clojure.string :as string]
+            [logseq.graph-parser.util.page-ref :as page-ref]
+            [frontend.util :as util]
+            [frontend.db.utils :as db-utils]
+            [frontend.db.async.util :as db-async-util]
+            [frontend.db.file-based.async :as file-async]))
+
+(def <q db-async-util/<q)
+
+(defn <get-files
+  [graph]
+  (p/let [result (<q
+                  graph
+                  '[:find ?path ?modified-at
+                    :where
+                    [?file :file/path ?path]
+                    [(get-else $ ?file :file/last-modified-at 0) ?modified-at]])]
+    (->> result seq reverse)))
+
+(defn <get-all-templates
+  [graph]
+  (p/let [result (<q graph
+                     '[:find ?t ?b
+                       :where
+                       [?b :block/properties ?p]
+                       [(get ?p :template) ?t]])]
+    (into {} result)))
+
+(defn <db-based-get-all-properties
+  ":block/type could be one of [property, class]."
+  [graph]
+  (<q graph
+      '[:find [?n ...]
+        :where
+        [?e :block/type "property"]
+        [?e :block/original-name ?n]]))
+
+(defn <get-all-properties
+  "Returns a seq of property name strings"
+  []
+  (when-let [graph (state/get-current-repo)]
+    (if (config/db-based-graph? graph)
+      (<db-based-get-all-properties graph)
+      (file-async/<file-based-get-all-properties graph))))
+
+(comment
+  (defn <get-pages
+    [graph]
+    (p/let [result (<q graph
+                       '[:find [?page-original-name ...]
+                         :where
+                         [?page :block/name ?page-name]
+                         [(get-else $ ?page :block/original-name ?page-name) ?page-original-name]])]
+      (remove db-model/hidden-page? result))))
+
+(defn <get-db-based-property-values
+  [graph property]
+  (let [property-name (if (keyword? property)
+                        (name property)
+                        (util/page-name-sanity-lc property))]
+    (p/let [result (<q graph
+                       '[:find ?prop-type ?v
+                         :in $ ?prop-name
+                         :where
+                         [?b :block/properties ?bp]
+                         [?prop-b :block/name ?prop-name]
+                         [?prop-b :block/uuid ?prop-uuid]
+                         [?prop-b :block/schema ?prop-schema]
+                         [(get ?prop-schema :type) ?prop-type]
+                         [(get ?bp ?prop-uuid) ?v]]
+                       property-name)]
+      (->> result
+           (map (fn [[prop-type v]] [prop-type (if (coll? v) v [v])]))
+           (mapcat (fn [[prop-type vals]]
+                     (case prop-type
+                       :default
+                       ;; Remove multi-block properties as there isn't a supported approach to query them yet
+                       (map str (remove uuid? vals))
+                       (:page :date)
+                       (map #(page-ref/->page-ref (:block/original-name (db-utils/entity graph [:block/uuid %])))
+                            vals)
+                       :number
+                       vals
+                       ;; Checkboxes returned as strings as builder doesn't display boolean values correctly
+                       (map str vals))))
+           ;; Remove blanks as they match on everything
+           (remove string/blank?)
+           (distinct)
+           (sort)))))
+
+(defn <get-property-values
+  [graph property]
+  (if (config/db-based-graph? graph)
+    (<get-db-based-property-values graph property)
+    (file-async/<get-file-based-property-values graph property)))

src/main/frontend/db/async/util.cljs

@@ -0,0 +1,12 @@
+(ns frontend.db.async.util
+  "Async util helper"
+  (:require [frontend.persist-db.browser :as db-browser]
+            [cljs-bean.core :as bean]
+            [promesa.core :as p]))
+
+(defn <q
+  [graph & inputs]
+  (assert (not-any? fn? inputs) "Async query inptus can't include fns because fn can't be serialized")
+  (when-let [sqlite @db-browser/*sqlite]
+    (p/let [result (.q sqlite graph (pr-str inputs))]
+      (bean/->clj result))))

src/main/frontend/db/file_based/async.cljs

@@ -0,0 +1,57 @@
+(ns frontend.db.file-based.async
+  "File based async queries"
+  (:require [promesa.core :as p]
+            [frontend.db.async.util :as db-async-util]
+            [clojure.string :as string]
+            [logseq.graph-parser.util.page-ref :as page-ref]))
+
+(def <q db-async-util/<q)
+
+(defn <file-based-get-all-properties
+  [graph]
+  (p/let [properties (<q graph
+                         '[:find [?p ...]
+                           :where
+                           [_ :block/properties ?p]])
+          properties (remove (fn [m] (empty? m)) properties)]
+    (->> (map keys properties)
+         (apply concat)
+         distinct
+         sort
+         (map name))))
+
+(defn- property-value-for-refs-and-text
+  "Given a property value's refs and full text, determines the value to
+  autocomplete"
+  [[refs text]]
+  (if (or (not (coll? refs)) (= 1 (count refs)))
+    text
+    (map #(cond
+            (string/includes? text (page-ref/->page-ref %))
+            (page-ref/->page-ref %)
+            (string/includes? text (str "#" %))
+            (str "#" %)
+            :else
+            %)
+         refs)))
+
+(defn <get-file-based-property-values
+  [graph property]
+  (p/let [result (<q graph
+                     '[:find ?property-val ?text-property-val
+                       :in $ ?property
+                       :where
+                       [?b :block/properties ?p]
+                       [?b :block/properties-text-values ?p2]
+                       [(get ?p ?property) ?property-val]
+                       [(get ?p2 ?property) ?text-property-val]]
+                     property)]
+    (->>
+     result
+     (map property-value-for-refs-and-text)
+     (map (fn [x] (if (coll? x) x [x])))
+     (apply concat)
+     (map str)
+     (remove string/blank?)
+     distinct
+     sort)))

src/main/frontend/db/model.cljs

@@ -17,7 +17,6 @@
             [logseq.db.frontend.rules :as rules]
             [logseq.graph-parser.config :as gp-config]
             [logseq.graph-parser.text :as text]
-            [logseq.graph-parser.util.page-ref :as page-ref]
             [logseq.graph-parser.util.db :as db-util]
             [logseq.graph-parser.util :as gp-util]
             [logseq.outliner.pipeline :as outliner-pipeline]
@@ -1186,130 +1185,6 @@ independent of format as format specific heading characters are stripped"
   [page-name]
   (:block/journal? (db-utils/entity [:block/name page-name])))
 
-;; This is a file graph only feature
-(defn get-all-templates
-  []
-  (let [pred (fn [_db properties]
-               (some? (:template properties)))]
-    (->> (d/q
-          '[:find ?b ?p
-            :in $ ?pred
-            :where
-            [?b :block/properties ?p]
-            [(?pred $ ?p)]]
-          (conn/get-db)
-          pred)
-         (map (fn [[e m]]
-                [(get m :template) e]))
-         (into {}))))
-
-(defn file-based-get-all-properties
-  []
-  (let [db (conn/get-db)
-        properties (d/q
-                    '[:find [?p ...]
-                      :where
-                      [_ :block/properties ?p]]
-                    db)
-        properties (remove (fn [m] (empty? m)) properties)]
-    (->> (map keys properties)
-         (apply concat)
-         distinct
-         sort)))
-
-(defn db-based-get-all-properties
-  ":block/type could be one of [property, class]."
-  []
-  (let [db (conn/get-db)
-        ids (->> (d/datoms db :aevt :block/schema)
-                 (map :e))]
-    (->> ids
-         (map db-utils/entity)
-         (filter #(contains? (:block/type %) "property"))
-         (map :block/original-name))))
-
-(defn get-all-properties
-  "Returns a seq of property name strings"
-  []
-  (if (config/db-based-graph? (state/get-current-repo))
-    (db-based-get-all-properties)
-    (map name (file-based-get-all-properties))))
-
-(defn- property-value-for-refs-and-text
-  "Given a property value's refs and full text, determines the value to
-  autocomplete"
-  [[refs text]]
-  (if (or (not (coll? refs)) (= 1 (count refs)))
-    text
-    (map #(cond
-            (string/includes? text (page-ref/->page-ref %))
-            (page-ref/->page-ref %)
-            (string/includes? text (str "#" %))
-            (str "#" %)
-            :else
-            %)
-         refs)))
-
-(defn get-property-values
-  [property]
-  (let [pred (fn [_db properties text-properties]
-               [(get properties property)
-                (get text-properties property)])]
-    (->>
-     (d/q
-      '[:find ?property-val ?text-property-val
-        :in $ ?pred
-        :where
-        [?b :block/properties ?p]
-        [?b :block/properties-text-values ?p2]
-        [(?pred $ ?p ?p2) [?property-val ?text-property-val]]]
-      (conn/get-db)
-      pred)
-     (map property-value-for-refs-and-text)
-     (map (fn [x] (if (coll? x) x [x])))
-     (apply concat)
-     (map str)
-     (remove string/blank?)
-     (distinct)
-     (sort))))
-
-(defn get-db-property-values
-  "Returns all property values of a given property for use in a simple query.
-   Property values that are references are displayed as page references"
-  [repo property]
-  (let [property-name (if (keyword? property)
-                        (name property)
-                        (util/page-name-sanity-lc property))]
-    (->> (d/q
-         '[:find ?prop-type ?v
-           :in $ ?prop-name
-           :where
-           [?b :block/properties ?bp]
-           [?prop-b :block/name ?prop-name]
-           [?prop-b :block/uuid ?prop-uuid]
-           [?prop-b :block/schema ?prop-schema]
-           [(get ?prop-schema :type) ?prop-type]
-           [(get ?bp ?prop-uuid) ?v]]
-         (conn/get-db repo)
-         property-name)
-        (map (fn [[prop-type v]] [prop-type (if (coll? v) v [v])]))
-        (mapcat (fn [[prop-type vals]]
-                  (case prop-type
-                    :default
-                   ;; Remove multi-block properties as there isn't a supported approach to query them yet
-                    (map str (remove uuid? vals))
-                    (:page :date)
-                    (map #(page-ref/->page-ref (:block/original-name (db-utils/entity repo [:block/uuid %])))
-                         vals)
-                    :number
-                    vals
-                   ;; Checkboxes returned as strings as builder doesn't display boolean values correctly
-                    (map str vals))))
-       ;; Remove blanks as they match on everything
-        (remove string/blank?)
-        (distinct)
-        (sort))))
-
 (defn get-block-property-values
   "Get blocks which have this property."
   [property-uuid]
@@ -1362,27 +1237,6 @@ independent of format as format specific heading characters are stripped"
            [?refed-b   :block/uuid ?refed-uuid]
            [?referee-b :block/refs ?refed-b]] db)))
 
-;; block/uuid and block/content
-(defn get-single-block-contents [id]
-  (let [e (db-utils/entity [:block/uuid id])]
-    (when-not (and (nil? (:block/name e))
-                   (string/blank? (:block/content e))) ; empty block
-      {:db/id (:db/id e)
-       :block/name (:block/name e)
-       :block/uuid id
-       :block/page (:db/id (:block/page e))
-       :block/content (:block/content e)
-       :block/format (:block/format e)
-       :block/properties (:block/properties e)})))
-
-(defn get-all-block-contents
-  []
-  (when-let [db (conn/get-db)]
-    (->> (d/datoms db :avet :block/uuid)
-         (map :v)
-         (map get-single-block-contents)
-         (remove nil?))))
-
 (defn delete-blocks
   [repo-url files _delete-page?]
   (when (seq files)

src/main/frontend/db_worker.cljs

@@ -254,14 +254,21 @@
    (when-let [conn (get-datascript-conn repo)]
      (:max-tx @conn)))
 
+  (q [_this repo inputs-str]
+     "Datascript q"
+     (when-let [conn (get-datascript-conn repo)]
+       (let [inputs (edn/read-string inputs-str)]
+         (let [result (apply d/q (first inputs) @conn (rest inputs))]
+           (bean/->js result)))))
+
   (transact
    [_this repo tx-data tx-meta]
    (when-let [conn (get-datascript-conn repo)]
      (try
        (let [tx-data (edn/read-string tx-data)
-             tx-meta (edn/read-string tx-meta)]
-         (d/transact! conn tx-data tx-meta)
-         nil)
+             tx-meta (edn/read-string tx-meta)
+             tx-report (d/transact! conn tx-data tx-meta)]
+         (search/sync-search-indice repo tx-report))
        (catch :default e
          (prn :debug :error)
          (js/console.error e)))))
@@ -323,6 +330,21 @@
      (search/truncate-table! db)
      nil))
 
+  (search-build-blocks-indice
+   [this repo]
+   (when-let [conn (get-datascript-conn repo)]
+     (search/build-blocks-indice repo @conn)))
+
+  (search-build-pages-indice
+   [this repo]
+   (when-let [conn (get-datascript-conn repo)]
+     (search/build-blocks-indice repo @conn)))
+
+  (page-search
+   [this repo q limit]
+   (when-let [conn (get-datascript-conn repo)]
+     (search/page-search repo @conn q limit)))
+
   (dangerousRemoveAllDbs
    [this repo]
    (p/let [dbs (.listDB this)]

src/main/frontend/handler/editor.cljs

@@ -1644,14 +1644,14 @@
       (when (>= pos 0)
         (text-util/wrapped-by? value pos before end)))))
 
-(defn get-matched-pages
+(defn <get-matched-pages
   "Return matched page names"
   [q]
-  (let [block (state/get-edit-block)
-        editing-page (and block
-                          (when-let [page-id (:db/id (:block/page block))]
-                            (:block/name (db/entity page-id))))
-        pages (search/page-search q)]
+  (p/let [block (state/get-edit-block)
+          editing-page (and block
+                            (when-let [page-id (:db/id (:block/page block))]
+                              (:block/name (db/entity page-id))))
+          pages (search/page-search q)]
     (if editing-page
       ;; To prevent self references
       (remove (fn [p] (= (util/page-name-sanity-lc p) editing-page)) pages)
@@ -1680,11 +1680,11 @@
          (contains? current-and-parents (:block/uuid h)))
        result))))
 
-(defn get-matched-templates
+(defn <get-matched-templates
   [q]
   (search/template-search q))
 
-(defn get-matched-properties
+(defn <get-matched-properties
   [q]
   (search/property-search q))
 

src/main/frontend/handler/events.cljs

@@ -808,6 +808,9 @@
    {:id :new-db-graph
     :label "graph-setup"}))
 
+(defmethod handle :search/transact-data [[_ repo data]]
+  (search/transact-blocks! repo data))
+
 (defmethod handle :class/configure [[_ page]]
   (state/set-modal!
    #(vector :<>

src/main/frontend/handler/page.cljs

@@ -6,6 +6,7 @@
             [frontend.config :as config]
             [frontend.date :as date]
             [frontend.db :as db]
+            [frontend.db.async :as db-async]
             [frontend.db.model :as model]
             [frontend.fs :as fs]
             [frontend.handler.common :as common-handler]
@@ -130,10 +131,11 @@
 (def rebuild-slash-commands-list!
   (debounce init-commands! 1500))
 
-(defn template-exists?
+(defn <template-exists?
   [title]
   (when title
-    (let [templates (keys (db/get-all-templates))]
+    (p/let [result (db-async/<get-all-templates (state/get-current-repo))
+            templates (keys result)]
       (when (seq templates)
         (let [templates (map string/lower-case templates)]
           (contains? (set templates) (string/lower-case title)))))))

src/main/frontend/handler/search.cljs

@@ -32,13 +32,15 @@
                         (:db/id (db/entity repo [:block/name (util/page-name-sanity-lc page-db-id)]))
                         page-db-id)
            opts (if page-db-id (assoc opts :page (str page-db-id)) opts)]
-       (p/let [blocks (search/block-search repo q opts)]
+       (p/let [blocks (search/block-search repo q opts)
+               pages (search/page-search q)
+               files (search/file-search q)]
          (let [result (merge
                        {:blocks blocks
                         :has-more? (= limit (count blocks))}
                        (when-not page-db-id
-                         {:pages (search/page-search q)
-                          :files (search/file-search q)}))
+                         {:pages pages
+                          :files files}))
                search-key (if more? :search/more-result :search/result)]
            (swap! state/state assoc search-key result)
            result))))))

src/main/frontend/modules/outliner/datascript.cljs

@@ -7,7 +7,6 @@
             [frontend.config :as config]
             [logseq.graph-parser.util :as gp-util]
             [lambdaisland.glogi :as log]
-            [frontend.search :as search]
             [clojure.string :as string]
             [frontend.util :as util]
             [logseq.graph-parser.util.block-ref :as block-ref]
@@ -61,9 +60,7 @@
     (when (or (:outliner/transact? tx-meta)
               (:outliner-op tx-meta)
               (:whiteboard/transact? tx-meta))
-      (undo-redo/listen-db-changes! tx-report))
-
-    (search/sync-search-indice! repo tx-report)))
+      (undo-redo/listen-db-changes! tx-report))))
 
 (defn- remove-nil-from-transaction
   [txs]

src/main/frontend/persist_db/browser.cljs

@@ -10,7 +10,8 @@
             [frontend.handler.notification :as notification]
             [cljs-bean.core :as bean]
             [frontend.state :as state]
-            [electron.ipc :as ipc]))
+            [electron.ipc :as ipc]
+            [frontend.handler.file-based.property.util :as property-util]))
 
 (defonce *sqlite (atom nil))
 
@@ -83,7 +84,20 @@
       (p/do!
        (ipc/ipc :db-transact repo tx-data' tx-meta')
        (if sqlite
-         (.transact sqlite repo tx-data' tx-meta')
+         (p/let [result (.transact sqlite repo tx-data' tx-meta')
+                 result' (bean/->clj result)
+                 file-based? (config/local-file-based-graph? repo)
+                 data (cond-> result'
+                        file-based?
+                        ;; remove built-in properties from content
+                        (update :blocks-to-add
+                                (fn [blocks]
+                                  (map #(update % :content
+                                               (fn [content]
+                                                 (property-util/remove-built-in-properties (get % :format :markdown) content)))
+                                    blocks))))]
+           (state/pub-event! [:search/transact-data repo data])
+           nil)
          (notification/show! "Latest change was not saved! Please restart the application." :error))
        nil)))
 

src/main/frontend/search.cljs

@@ -1,92 +1,28 @@
 (ns frontend.search
   "Provides search functionality for a number of features including Cmd-K
   search. Most of these fns depend on the search protocol"
-  (:require [cljs-bean.core :as bean]
-            [clojure.string :as string]
-            [logseq.graph-parser.config :as gp-config]
-            [frontend.db :as db]
-            [frontend.db.model :as db-model]
+  (:require [clojure.string :as string]
             [frontend.search.agency :as search-agency]
-            [frontend.search.db :as search-db :refer [indices]]
             [frontend.search.protocol :as protocol]
             [frontend.state :as state]
             [frontend.util :as util]
-            [goog.object :as gobj]
             [promesa.core :as p]
-            [datascript.core :as d]
-            [frontend.handler.file-based.property.util :as property-util]
+            [frontend.search.browser :as search-browser]
+            [frontend.search.fuzzy :as fuzzy]
+            [logseq.graph-parser.config :as gp-config]
+            [frontend.db.async :as db-async]
             [frontend.config :as config]
-            [logseq.db.frontend.property :as db-property]))
+            [logseq.db.frontend.property :as db-property]
+            [frontend.handler.file-based.property.util :as property-util]
+            [frontend.db.model :as db-model]
+            [cljs-bean.core :as bean]))
+
+(def fuzzy-search fuzzy/fuzzy-search)
 
 (defn get-engine
   [repo]
   (search-agency/->Agency repo))
 
-;; Copied from https://gist.github.com/vaughnd/5099299
-(defn str-len-distance
-  ;; normalized multiplier 0-1
-  ;; measures length distance between strings.
-  ;; 1 = same length
-  [s1 s2]
-  (let [c1 (count s1)
-        c2 (count s2)
-        maxed (max c1 c2)
-        mined (min c1 c2)]
-    (double (- 1
-               (/ (- maxed mined)
-                  maxed)))))
-
-(def MAX-STRING-LENGTH 1000.0)
-
-(defn clean-str
-  [s]
-  (string/replace (string/lower-case s) #"[\[ \\/_\]\(\)]+" ""))
-
-(defn char-array
-  [s]
-  (bean/->js (seq s)))
-
-(defn score
-  [oquery ostr]
-  (let [query (clean-str oquery)
-        str (clean-str ostr)]
-    (loop [q (seq (char-array query))
-           s (seq (char-array str))
-           mult 1
-           idx MAX-STRING-LENGTH
-           score 0]
-      (cond
-        ;; add str-len-distance to score, so strings with matches in same position get sorted by length
-        ;; boost score if we have an exact match including punctuation
-        (empty? q) (+ score
-                      (str-len-distance query str)
-                      (if (<= 0 (.indexOf ostr oquery)) MAX-STRING-LENGTH 0))
-        (empty? s) 0
-        :else (if (= (first q) (first s))
-                  (recur (rest q)
-                         (rest s)
-                         (inc mult) ;; increase the multiplier as more query chars are matched
-                         (dec idx) ;; decrease idx so score gets lowered the further into the string we match
-                         (+ mult score)) ;; score for this match is current multiplier * idx
-                  (recur q
-                         (rest s)
-                         1 ;; when there is no match, reset multiplier to one
-                         (dec idx)
-                         score))))))
-
-(defn fuzzy-search
-  [data query & {:keys [limit extract-fn]
-                 :or {limit 20}}]
-  (let [query (util/search-normalize query (state/enable-search-remove-accents?))]
-    (->> (take limit
-               (sort-by :score (comp - compare)
-                        (filter #(< 0 (:score %))
-                                (for [item data]
-                                  (let [s (str (if extract-fn (extract-fn item) item))]
-                                    {:data item
-                                     :score (score query (util/search-normalize s (state/enable-search-remove-accents?)))})))))
-         (map :data))))
-
 (defn block-search
   [repo q option]
   (when-let [engine (get-engine repo)]
@@ -94,176 +30,80 @@
       (when-not (string/blank? q)
         (protocol/query engine q option)))))
 
-(defn- transact-blocks!
-  [repo data]
-  (when-let [engine (get-engine repo)]
-    (protocol/transact-blocks! engine data)))
-
-(defn exact-matched?
-  "Check if two strings points toward same search result"
-  [q match]
-  (when (and (string? q) (string? match))
-    (boolean
-     (reduce
-      (fn [coll char]
-        (let [coll' (drop-while #(not= char %) coll)]
-          (if (seq coll')
-            (rest coll')
-            (reduced false))))
-      (seq (util/search-normalize match (state/enable-search-remove-accents?)))
-      (seq (util/search-normalize q (state/enable-search-remove-accents?)))))))
-
 (defn page-search
-  "Return a list of page names that match the query"
   ([q]
    (page-search q 100))
   ([q limit]
-   (when-let [repo (state/get-current-repo)]
-     (let [q (util/search-normalize q (state/enable-search-remove-accents?))
-           q (clean-str q)
-           q (if (= \# (first q)) (subs q 1) q)]
-       (when-not (string/blank? q)
-         (let [indice (or (get-in @indices [repo :pages])
-                          (search-db/make-pages-title-indice!))
-               result (->> (.search indice q (clj->js {:limit limit}))
-                           (bean/->clj))]
-           (->> result
-                (util/distinct-by (fn [i] (string/trim (get-in i [:item :name]))))
-                (map
-                 (fn [{:keys [item]}]
-                   (:original-name item)))
-                (remove nil?)
-                (map string/trim)
-                (distinct)
-                (filter (fn [original-name]
-                          (exact-matched? q original-name))))))))))
+   (when-let [^js sqlite @search-browser/*sqlite]
+     (p/let [result (.page-search sqlite (state/get-current-repo) q limit)]
+       (bean/->clj result)))))
 
 (defn file-search
   ([q]
    (file-search q 3))
   ([q limit]
-   (let [q (clean-str q)]
-     (when-not (string/blank? q)
-       (let [mldoc-exts (set (map name gp-config/mldoc-support-formats))
-             files (->> (db/get-files (state/get-current-repo))
-                        (map first)
-                        (remove (fn [file]
-                                  (mldoc-exts (util/get-file-ext file)))))]
-         (when (seq files)
-           (fuzzy-search files q :limit limit)))))))
+   (when-let [repo (state/get-current-repo)]
+     (let [q (fuzzy/clean-str q)]
+      (when-not (string/blank? q)
+        (p/let [mldoc-exts (set (map name gp-config/mldoc-support-formats))
+                result (db-async/<get-files repo)
+                files (->> result
+                           (map first)
+                           (remove (fn [file]
+                                     (mldoc-exts (util/get-file-ext file)))))]
+          (when (seq files)
+            (fuzzy/fuzzy-search files q :limit limit))))))))
 
 (defn template-search
   ([q]
    (template-search q 100))
   ([q limit]
-   (when q
-     (let [q (clean-str q)
-           templates (db/get-all-templates)]
-       (when (seq templates)
-         (let [result (fuzzy-search (keys templates) q :limit limit)]
-           (vec (select-keys templates result))))))))
+   (when-let [repo (state/get-current-repo)]
+     (when q
+       (p/let [q (fuzzy/clean-str q)
+               templates (db-async/<get-all-templates repo)]
+         (when (seq templates)
+           (let [result (fuzzy/fuzzy-search (keys templates) q {:limit limit})]
+             (vec (select-keys templates result)))))))))
 
 (defn get-all-properties
   []
-  (let [hidden-props (if (config/db-based-graph? (state/get-current-repo))
-                       (set (map #(or (get-in db-property/built-in-properties [% :original-name])
-                                      (name %))
-                                 db-property/hidden-built-in-properties))
-                       (set (map name (property-util/hidden-properties))))]
-    (remove hidden-props (db-model/get-all-properties))))
+  (when-let [repo (state/get-current-repo)]
+    (let [hidden-props (if (config/db-based-graph? repo)
+                        (set (map #(or (get-in db-property/built-in-properties [% :original-name])
+                                       (name %))
+                                  db-property/hidden-built-in-properties))
+                        (set (map name (property-util/hidden-properties))))]
+     (p/let [properties (db-async/<get-all-properties)]
+       (remove hidden-props properties)))))
 
 (defn property-search
   ([q]
    (property-search q 100))
   ([q limit]
    (when q
-     (let [q (clean-str q)
-           properties (get-all-properties)]
+     (p/let [q (fuzzy/clean-str q)
+             properties (get-all-properties)]
        (when (seq properties)
          (if (string/blank? q)
            properties
-           (let [result (fuzzy-search properties q :limit limit)]
+           (let [result (fuzzy/fuzzy-search properties q :limit limit)]
              (vec result))))))))
 
+;; file-based graph only
 (defn property-value-search
   ([property q]
    (property-value-search property q 100))
   ([property q limit]
-   (when q
-     (let [q (clean-str q)
-           result (db-model/get-property-values (keyword property))]
-       (when (seq result)
-         (if (string/blank? q)
-           result
-           (let [result (fuzzy-search result q :limit limit)]
-             (vec result))))))))
-
-(defn- get-blocks-from-datoms-impl
-  [{:keys [db-after db-before]} datoms]
-  (when (seq datoms)
-    (let [blocks-to-add-set (->> (filter :added datoms)
-                                 (map :e)
-                                 (set))
-          blocks-to-remove-set (->> (remove :added datoms)
-                                    (filter #(= :block/uuid (:a %)))
-                                    (map :e)
-                                    (set))
-          blocks-to-add-set' (if (and (config/db-based-graph? (state/get-current-repo)) (seq blocks-to-add-set))
-                               (->> blocks-to-add-set
-                                    (mapcat (fn [id] (map :db/id (:block/_refs (db/entity id)))))
-                                    (concat blocks-to-add-set)
-                                    set)
-                               blocks-to-add-set)]
-      {:blocks-to-remove     (->>
-                              (map #(d/entity db-before %) blocks-to-remove-set)
-                              (remove nil?)
-                              (remove db-model/hidden-page?))
-       :blocks-to-add        (->>
-                              (map #(d/entity db-after %) blocks-to-add-set')
-                              (remove nil?)
-                              (remove db-model/hidden-page?))})))
-
-(defn- get-direct-blocks-and-pages
-  [tx-report]
-  (let [data (:tx-data tx-report)
-        datoms (filter
-                (fn [datom]
-                  ;; Capture any direct change on page display title, page ref or block content
-                  (contains? #{:block/uuid :block/name :block/original-name :block/content :block/properties :block/schema} (:a datom)))
-                data)]
-    (when (seq datoms)
-      (get-blocks-from-datoms-impl tx-report datoms))))
-
-;; TODO merge with logic in `invoke-hooks` when feature and test is sufficient
-(defn sync-search-indice!
-  [repo tx-report]
-  (let [{:keys [blocks-to-add blocks-to-remove]} (get-direct-blocks-and-pages tx-report)]
-    ;; TODO: remove this once we have fuzzy search support on SQLite
-    ;; update page title indice
-    (let [pages-to-add (filter :block/name blocks-to-add)
-          pages-to-remove (filter :block/name blocks-to-remove)]
-      (when (or (seq pages-to-add) (seq pages-to-remove))
-        (swap! search-db/indices update-in [repo :pages]
-               (fn [indice]
-                 (when indice
-                   (doseq [page-entity pages-to-remove]
-                     (.remove indice
-                              (fn [page]
-                                (= (:block/name page-entity)
-                                   (util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
-                   (doseq [page pages-to-add]
-                     (.add indice (bean/->js (search-db/original-page-name->index
-                                              (or (:block/original-name page)
-                                                  (:block/name page))))))
-                   indice)))))
-
-    ;; update block indice
-    (when (or (seq blocks-to-add) (seq blocks-to-remove))
-      (let [blocks-to-add (remove nil? (map search-db/block->index blocks-to-add))
-            blocks-to-remove (set (map (comp str :block/uuid) blocks-to-remove))]
-        (transact-blocks! repo
-                          {:blocks-to-remove-set blocks-to-remove
-                           :blocks-to-add        blocks-to-add})))))
+   (when-let [repo (state/get-current-repo)]
+     (when q
+      (let [q (fuzzy/clean-str q)
+            result (db-async/<get-property-values repo (keyword property))]
+        (when (seq result)
+          (if (string/blank? q)
+            result
+            (let [result (fuzzy/fuzzy-search result q :limit limit)]
+              (vec result)))))))))
 
 (defn rebuild-indices!
   ([]
@@ -271,20 +111,21 @@
   ([repo]
    (when repo
      (when-let [engine (get-engine repo)]
-       (let [page-titles (search-db/make-pages-title-indice!)]
-         (p/let [_ (protocol/rebuild-blocks-indice! engine)]
-           (let [result {:pages         page-titles ;; TODO: rename key to :page-titles
-                         }]
-             (swap! indices assoc repo result)
-             indices)))))))
+       (p/do!
+        (protocol/rebuild-pages-indice! engine)
+        (protocol/rebuild-blocks-indice! engine))))))
 
 (defn reset-indice!
   [repo]
   (when-let [engine (get-engine repo)]
-    (protocol/truncate-blocks! engine))
-  (swap! indices assoc-in [repo :pages] nil))
+    (protocol/truncate-blocks! engine)))
 
 (defn remove-db!
   [repo]
   (when-let [engine (get-engine repo)]
     (protocol/remove-db! engine)))
+
+(defn transact-blocks!
+  [repo data]
+  (when-let [engine (get-engine repo)]
+    (protocol/transact-blocks! engine data)))

src/main/frontend/search/agency.cljs

@@ -31,6 +31,12 @@
         (protocol/rebuild-blocks-indice! e))
       (protocol/rebuild-blocks-indice! e1)))
 
+  (rebuild-pages-indice! [_this]
+    (let [[e1 e2] (get-registered-engines repo)]
+      (doseq [e e2]
+        (protocol/rebuild-pages-indice! e))
+      (protocol/rebuild-pages-indice! e1)))
+
   (transact-blocks! [_this data]
     (doseq [e (get-flatten-registered-engines repo)]
       (protocol/transact-blocks! e data)))

src/main/frontend/search/browser.cljs

@@ -5,7 +5,8 @@
             [promesa.core :as p]
             [frontend.persist-db.browser :as browser]
             [frontend.state :as state]
-            [frontend.search.db :as search-db]))
+            [frontend.config :as config]
+            [frontend.handler.file-based.property.util :as property-util]))
 
 (defonce *sqlite browser/*sqlite)
 
@@ -20,10 +21,24 @@
                  :block/content content
                  :block/page (uuid page)}) result))
       (p/resolved nil)))
+  (rebuild-pages-indice! [_this]
+    (if-let [^js sqlite @*sqlite]
+      (.search-build-pages-indice sqlite repo)
+      (p/resolved nil)))
   (rebuild-blocks-indice! [this]
     (if-let [^js sqlite @*sqlite]
-      (p/let [_ (protocol/truncate-blocks! this)
-              blocks (search-db/build-blocks-indice)
+      (p/let [repo (state/get-current-repo)
+              file-based? (config/local-file-based-graph? repo)
+              _ (protocol/truncate-blocks! this)
+              result (.search-build-blocks-indice sqlite repo)
+              blocks (cond->> (bean/->clj result)
+                       file-based?
+                       ;; remove built-in properties from content
+                       (map #(update % :content
+                                     (fn [content]
+                                       (property-util/remove-built-in-properties (get % :format :markdown) content))))
+                       true
+                       bean/->js)
               _ (when (seq blocks)
                   (.search-upsert-blocks sqlite repo blocks))])
       (p/resolved nil)))

src/main/frontend/search/db.cljs

@@ -1,118 +0,0 @@
-(ns ^:no-doc frontend.search.db
-  (:require [cljs-bean.core :as bean]
-            [clojure.string :as string]
-            [frontend.db :as db]
-            [frontend.db.model :as model]
-            [frontend.handler.db-based.property.util :as db-pu]
-            [frontend.state :as state]
-            [frontend.config :as config]
-            [frontend.util :as util]
-            ["fuse.js" :as fuse]
-            [frontend.handler.file-based.property.util :as property-util]))
-
-;; Notice: When breaking changes happen, bump version in src/electron/electron/search.cljs
-
-(defonce indices (atom nil))
-
-(defn- max-len
-  []
-  (state/block-content-max-length (state/get-current-repo)))
-
-(defn- sanitize
-  [content]
-  (some-> content
-          (util/search-normalize (state/enable-search-remove-accents?))))
-
-(defn- get-db-properties-str
-  "Similar to db-pu/readable-properties but with a focus on making property values searchable"
-  [properties]
-  (->> properties
-       (map
-        (fn [[k v]]
-          (let [values
-                (->> (if (set? v) v #{v})
-                     (map (fn [val]
-                            (if (uuid? val)
-                              (let [e (db/entity [:block/uuid val])
-                                    value (or
-                                           ;; closed value
-                                           (db-pu/property-value-when-closed e)
-                                           ;; page
-                                           (:block/original-name e)
-                                           ;; block generated by template
-                                           (and
-                                            (get-in e [:block/metadata :created-from-template])
-                                            (:block/content e))
-                                           ;; first child
-                                           (let [parent-id (:db/id e)]
-                                             (:block/content (model/get-by-parent-&-left (db/get-db) parent-id parent-id))))]
-                                value)
-                              val)))
-                     (remove string/blank?))]
-            (when (seq values)
-              (str (:block/original-name (db/entity [:block/uuid k]))
-                   ": "
-                   (string/join "; " values))))))
-       (remove nil?)
-       (string/join ";; ")))
-
-(defn block->index
-  "Convert a block to the index for searching"
-  [{:block/keys [name uuid page content properties format]
-    :or {format :markdown}
-    :as block}]
-  (let [repo (state/get-current-repo)
-        page? (some? name)
-        block? (nil? name)
-        db-based? (config/db-based-graph? repo)]
-    (when-not (or
-               (and page? name (model/whiteboard-page? name))
-               (and block? (> (count content) (max-len)))
-               (and (empty? properties)
-                    (or (and block? (string/blank? content))
-                        (and db-based? page?))))        ; empty page or block
-      (let [content (if block?
-                      (if db-based? content (property-util/remove-built-in-properties format content))
-                        ;; File based page content
-                      (if db-based?
-                        ""            ; empty page content
-                        (some-> (:block/file (db/entity (:db/id block))) :file/content)))
-            content' (if (and db-based? (seq properties))
-                       (str content (when (not= content "") "\n") (get-db-properties-str properties))
-                       content)]
-        (when-not (string/blank? content')
-          {:id (str uuid)
-           :page (str (:block/uuid page))
-           :content (sanitize content')})))))
-
-(defn original-page-name->index
-  [p]
-  (when p
-    {:name (util/search-normalize p (state/enable-search-remove-accents?))
-     :original-name p}))
-
-(defn make-pages-title-indice!
-  "Build a page title indice from scratch.
-   Incremental page title indice is implemented in frontend.search.sync-search-indice!
-   Rename from the page indice since 10.25.2022, since this is only used for page title search.
-   From now on, page indice is talking about page content search."
-  []
-  (when-let [repo (state/get-current-repo)]
-    (let [pages (->> (db/get-pages (state/get-current-repo))
-                     (remove string/blank?)
-                     (map original-page-name->index)
-                     (bean/->js))
-          indice (fuse. pages
-                        (clj->js {:keys ["name"]
-                                  :shouldSort true
-                                  :tokenize true
-                                  :minMatchCharLength 1}))]
-      (swap! indices assoc-in [repo :pages] indice)
-      indice)))
-
-(defn build-blocks-indice
-  []
-  (->> (db/get-all-block-contents)
-       (map block->index)
-       (remove nil?)
-       (bean/->js)))

src/main/frontend/search/fuzzy.cljs

@@ -0,0 +1,70 @@
+(ns frontend.search.fuzzy
+  "fuzzy search"
+  (:require [clojure.string :as string]
+            [cljs-bean.core :as bean]
+            [frontend.worker.util :as util]))
+
+(def MAX-STRING-LENGTH 1000.0)
+
+(defn clean-str
+  [s]
+  (string/replace (string/lower-case s) #"[\[ \\/_\]\(\)]+" ""))
+
+(defn char-array
+  [s]
+  (bean/->js (seq s)))
+
+;; Copied from https://gist.github.com/vaughnd/5099299
+(defn str-len-distance
+  ;; normalized multiplier 0-1
+  ;; measures length distance between strings.
+  ;; 1 = same length
+  [s1 s2]
+  (let [c1 (count s1)
+        c2 (count s2)
+        maxed (max c1 c2)
+        mined (min c1 c2)]
+    (double (- 1
+               (/ (- maxed mined)
+                  maxed)))))
+
+(defn score
+  [oquery ostr]
+  (let [query (clean-str oquery)
+        str (clean-str ostr)]
+    (loop [q (seq (char-array query))
+           s (seq (char-array str))
+           mult 1
+           idx MAX-STRING-LENGTH
+           score 0]
+      (cond
+        ;; add str-len-distance to score, so strings with matches in same position get sorted by length
+        ;; boost score if we have an exact match including punctuation
+        (empty? q) (+ score
+                      (str-len-distance query str)
+                      (if (<= 0 (.indexOf ostr oquery)) MAX-STRING-LENGTH 0))
+        (empty? s) 0
+        :else (if (= (first q) (first s))
+                  (recur (rest q)
+                         (rest s)
+                         (inc mult) ;; increase the multiplier as more query chars are matched
+                         (dec idx) ;; decrease idx so score gets lowered the further into the string we match
+                         (+ mult score)) ;; score for this match is current multiplier * idx
+                  (recur q
+                         (rest s)
+                         1 ;; when there is no match, reset multiplier to one
+                         (dec idx)
+                         score))))))
+
+(defn fuzzy-search
+  [data query & {:keys [limit extract-fn]
+                 :or {limit 20}}]
+  (let [query (util/search-normalize query true)]
+    (->> (take limit
+               (sort-by :score (comp - compare)
+                        (filter #(< 0 (:score %))
+                                (for [item data]
+                                  (let [s (str (if extract-fn (extract-fn item) item))]
+                                    {:data item
+                                     :score (score query (util/search-normalize s true))})))))
+         (map :data))))

src/main/frontend/search/plugin.cljs

@@ -23,12 +23,14 @@
   (query [_this q opts]
     (call-service! service "search:query" (merge {:q q} opts) true))
 
-
   (rebuild-blocks-indice! [_this]
    ;; Not pushing all data for performance temporarily
    ;;(let [blocks (search-db/build-blocks-indice repo)])
     (call-service! service "search:rebuildBlocksIndice" {}))
 
+  (rebuild-pages-indice! [_this]
+    (call-service! service "search:rebuildPagesIndice" {}))
+
   (transact-blocks! [_this data]
     (let [{:keys [blocks-to-remove-set blocks-to-add]} data]
       (call-service! service "search:transactBlocks"

src/main/frontend/search/protocol.cljs

@@ -3,6 +3,7 @@
 (defprotocol Engine
   (query [this q option])
   (rebuild-blocks-indice! [this]) ;; TODO: rename to rebuild-indice!
+  (rebuild-pages-indice! [this]) ;; TODO: rename to rebuild-indice!
   (transact-blocks! [this data])
   (truncate-blocks! [this]) ;; TODO: rename to truncate-indice!
   (remove-db! [this]))

src/main/frontend/state.cljs

@@ -972,7 +972,9 @@ Similar to re-frame subscriptions"
              (gobj/get "id")))
    (when-let [elem js/document.activeElement]
      (when (util/input? elem)
-       (gobj/get elem "id")))))
+       (let [id (gobj/get elem "id")]
+         (when (string/starts-with? id "edit-block-")
+           id))))))
 
 (defn get-input
   []

src/main/frontend/util.cljc

@@ -8,7 +8,6 @@
             ["@capacitor/status-bar" :refer [^js StatusBar Style]]
             ["@capgo/capacitor-navigation-bar" :refer [^js NavigationBar]]
             ["grapheme-splitter" :as GraphemeSplitter]
-            ["remove-accents" :as removeAccents]
             ["sanitize-filename" :as sanitizeFilename]
             ["check-password-strength" :refer [passwordStrength]]
             ["path-complete-extname" :as pathCompleteExtname]
@@ -28,8 +27,8 @@
             [rum.core :as rum]
             [clojure.core.async :as async]
             [cljs.core.async.impl.channels :refer [ManyToManyChannel]]
-            [medley.core :as medley]
-            [frontend.pubsub :as pubsub]))
+            [frontend.pubsub :as pubsub]
+            [frontend.worker.util :as worker-util]))
   #?(:cljs (:import [goog.async Debouncer]))
   (:require
    [clojure.pprint]
@@ -76,23 +75,11 @@
   (string/join "/" parts))
 
 #?(:cljs
-   (defn safe-re-find
-     {:malli/schema [:=> [:cat :any :string] [:or :nil :string [:vector [:maybe :string]]]]}
-     [pattern s]
-     (when-not (string? s)
-       ;; TODO: sentry
-       (js/console.trace))
-     (when (string? s)
-       (re-find pattern s))))
+   (def safe-re-find worker-util/safe-re-find))
 
 #?(:cljs
    (do
-     (def uuid-pattern "[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}")
-     (defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
-     (defn uuid-string?
-       {:malli/schema [:=> [:cat :string] :boolean]}
-       [s]
-       (boolean (safe-re-find exactly-uuid-pattern s)))
+     (def uuid-string? worker-util/uuid-string?)
      (defn check-password-strength
        {:malli/schema [:=> [:cat :string] [:maybe
                                            [:map
@@ -594,9 +581,7 @@
 
 
 #?(:cljs
-   (defn distinct-by
-     [f col]
-     (medley/distinct-by f (seq col))))
+   (def distinct-by worker-util/distinct-by))
 
 #?(:cljs
    (defn distinct-by-last-wins
@@ -1034,14 +1019,7 @@
      (some-> string str (js/encodeURIComponent) (.replace "+" "%20"))))
 
 #?(:cljs
-   (defn search-normalize
-     "Normalize string for searching (loose)"
-     [s remove-accents?]
-     (when s
-       (let [normalize-str (.normalize (string/lower-case s) "NFKC")]
-         (if remove-accents?
-           (removeAccents normalize-str)
-           normalize-str)))))
+   (def search-normalize worker-util/search-normalize))
 
 #?(:cljs
    (def page-name-sanity-lc
@@ -1049,10 +1027,7 @@
      gp-util/page-name-sanity-lc))
 
 #?(:cljs
-   (defn safe-page-name-sanity-lc
-     [s]
-     (if (string? s)
-       (page-name-sanity-lc s) s)))
+   (def safe-page-name-sanity-lc worker-util/safe-page-name-sanity-lc))
 
 (defn get-page-original-name
   [page]

src/main/frontend/worker/search.cljs

@@ -1,9 +1,23 @@
 (ns frontend.worker.search
-  "SQLite search"
+  "Full-text and fuzzy search"
   (:require [clojure.string :as string]
             [promesa.core :as p]
-            [medley.core :as medley]
-            [cljs-bean.core :as bean]))
+            [cljs-bean.core :as bean]
+            ["fuse.js" :as fuse]
+            [goog.object :as gobj]
+            [datascript.core :as d]
+            [frontend.search.fuzzy :as fuzzy]
+            [frontend.worker.util :as util]))
+
+(defonce db-version-prefix "logseq_db_")
+(defn db-based-graph?
+  [s]
+  (boolean
+   (and (string? s)
+        (string/starts-with? s db-version-prefix))))
+
+;; TODO: use sqlite for fuzzy search
+(defonce indices (atom nil))
 
 (defn- add-blocks-fts-triggers!
   "Table bindings of blocks tables and the blocks FTS virtual tables"
@@ -125,10 +139,6 @@
       (string/replace match-input "," "")
       (str "\"" match-input "\""))))
 
-(defn distinct-by
-  [f col]
-  (medley/distinct-by f (seq col)))
-
 (defn search-blocks
   ":page - the page to specifically search on"
   [db q {:keys [limit page]}]
@@ -158,10 +168,286 @@
                                     :page page})))]
       (->>
        all-result
-       (distinct-by :uuid)
+       (util/distinct-by :uuid)
        (take limit)))))
 
 (defn truncate-table!
   [db]
   (.exec db "delete from blocks")
   (.exec db "delete from blocks_fts"))
+
+(defn- sanitize
+  [content]
+  (some-> content
+          (util/search-normalize true)))
+
+(defn- property-value-when-closed
+  "Returns property value if the given entity is type 'closed value' or nil"
+  [ent]
+  (when (contains? (:block/type ent) "closed value")
+    (get-in ent [:block/schema :value])))
+
+(defn get-by-parent-&-left
+  [db parent-id left-id]
+  (when (and parent-id left-id)
+    (let [lefts (:block/_left (d/entity db left-id))]
+      (some (fn [node] (when (and (= parent-id (:db/id (:block/parent node)))
+                                  (not= parent-id (:db/id node)))
+                         node)) lefts))))
+
+(defn- get-db-properties-str
+  "Similar to db-pu/readable-properties but with a focus on making property values searchable"
+  [db properties]
+  (->> properties
+       (map
+        (fn [[k v]]
+          (let [values
+                (->> (if (set? v) v #{v})
+                     (map (fn [val]
+                            (if (uuid? val)
+                              (let [e (d/entity db [:block/uuid val])
+                                    value (or
+                                           ;; closed value
+                                           (property-value-when-closed e)
+                                           ;; page
+                                           (:block/original-name e)
+                                           ;; block generated by template
+                                           (and
+                                            (get-in e [:block/metadata :created-from-template])
+                                            (:block/content e))
+                                           ;; first child
+                                           (let [parent-id (:db/id e)]
+                                             (:block/content (get-by-parent-&-left db parent-id parent-id))))]
+                                value)
+                              val)))
+                     (remove string/blank?))]
+            (when (seq values)
+              (str (:block/original-name (d/entity db [:block/uuid k]))
+                   ": "
+                   (string/join "; " values))))))
+       (remove nil?)
+       (string/join ";; ")))
+
+(defn whiteboard-page?
+  "Given a page name or a page object, check if it is a whiteboard page"
+  [db page]
+  (cond
+    (string? page)
+    (let [page (d/entity db [:block/name page])]
+      (or
+       (= (:block/type page) "whiteboard")
+       (contains? (set (:block/type page)) "whiteboard")))
+
+    (seq page)
+    (contains? (set (:block/type page)) "whiteboard")
+
+    :else false))
+
+(defn block->index
+  "Convert a block to the index for searching"
+  [repo db {:block/keys [name uuid page content properties format]
+            :as block}]
+  (let [page? (some? name)
+        block? (nil? name)
+        db-based? (db-based-graph? repo)]
+    (when-not (or
+               (and page? name (whiteboard-page? db name))
+               (and block? (> (count content) 10000))
+               (and (empty? properties)
+                    (or (and block? (string/blank? content))
+                        (and db-based? page?))))        ; empty page or block
+      (let [content (if block?
+                      content
+                      ;; File based page content
+                      (if db-based?
+                        ""            ; empty page content
+                        (some-> (:block/file (d/entity db (:db/id block))) :file/content)))
+            content' (if (and db-based? (seq properties))
+                       (str content (when (not= content "") "\n") (get-db-properties-str db properties))
+                       content)]
+        (when-not (string/blank? content')
+          {:id (str uuid)
+           :page (str (:block/uuid page))
+           :content (sanitize content')
+           :format format})))))
+
+(defn get-single-block-contents [db id]
+  (let [e (d/entity db [:block/uuid id])]
+    (when-not (and (nil? (:block/name e))
+                   (string/blank? (:block/content e))) ; empty block
+      {:db/id (:db/id e)
+       :block/name (:block/name e)
+       :block/uuid id
+       :block/page (:db/id (:block/page e))
+       :block/content (:block/content e)
+       :block/format (:block/format e)
+       :block/properties (:block/properties e)})))
+
+(defn get-all-block-contents
+  [db]
+  (when db
+    (->> (d/datoms db :avet :block/uuid)
+         (map :v)
+         (map #(get-single-block-contents db %))
+         (remove nil?))))
+
+(defn build-blocks-indice
+  [repo db]
+  (->> (get-all-block-contents db)
+       (map #(block->index repo db %))
+       (remove nil?)
+       (bean/->js)))
+
+(defn original-page-name->index
+  [p]
+  (when p
+    {:name (util/search-normalize p true)
+     :original-name p}))
+
+(defn- safe-subs
+  ([s start]
+   (let [c (count s)]
+     (safe-subs s start c)))
+  ([s start end]
+   (let [c (count s)]
+     (subs s (min c start) (min c end)))))
+
+(defn- hidden-page?
+  [page]
+  (when page
+    (if (string? page)
+      (and (string/starts-with? page "$$$")
+           (util/uuid-string? (safe-subs page 3)))
+      (contains? (set (:block/type page)) "hidden"))))
+
+(defn get-all-pages
+  [db]
+  (->>
+   (d/q
+    '[:find [?page-original-name ...]
+      :where
+      [?page :block/name ?page-name]
+      [(get-else $ ?page :block/original-name ?page-name) ?page-original-name]]
+    db)
+   (remove hidden-page?)))
+
+(defn build-page-indice
+  "Build a page title indice from scratch.
+   Incremental page title indice is implemented in frontend.search.sync-search-indice!
+   Rename from the page indice since 10.25.2022, since this is only used for page title search.
+   From now on, page indice is talking about page content search."
+  [repo db]
+  (let [pages (->> (get-all-pages db)
+                   (remove string/blank?)
+                   (map original-page-name->index)
+                   (bean/->js))
+        indice (fuse. pages
+                      (clj->js {:keys ["name"]
+                                :shouldSort true
+                                :tokenize true
+                                :minMatchCharLength 1}))]
+    (swap! indices assoc-in [repo :pages] indice)
+    indice))
+
+(defn- get-blocks-from-datoms-impl
+  [repo {:keys [db-after db-before]} datoms]
+  (when (seq datoms)
+    (let [blocks-to-add-set (->> (filter :added datoms)
+                                 (map :e)
+                                 (set))
+          blocks-to-remove-set (->> (remove :added datoms)
+                                    (filter #(= :block/uuid (:a %)))
+                                    (map :e)
+                                    (set))
+          blocks-to-add-set' (if (and (db-based-graph? repo) (seq blocks-to-add-set))
+                               (->> blocks-to-add-set
+                                    (mapcat (fn [id] (map :db/id (:block/_refs (d/entity db-after id)))))
+                                    (concat blocks-to-add-set)
+                                    set)
+                               blocks-to-add-set)]
+      {:blocks-to-remove     (->>
+                              (map #(d/entity db-before %) blocks-to-remove-set)
+                              (remove nil?)
+                              (remove hidden-page?))
+       :blocks-to-add        (->>
+                              (map #(d/entity db-after %) blocks-to-add-set')
+                              (remove nil?)
+                              (remove hidden-page?))})))
+
+(defn- get-direct-blocks-and-pages
+  [repo tx-report]
+  (let [data (:tx-data tx-report)
+        datoms (filter
+                (fn [datom]
+                  ;; Capture any direct change on page display title, page ref or block content
+                  (contains? #{:block/uuid :block/name :block/original-name :block/content :block/properties :block/schema} (:a datom)))
+                data)]
+    (when (seq datoms)
+      (get-blocks-from-datoms-impl repo tx-report datoms))))
+
+(defn sync-search-indice
+  [repo tx-report]
+  (let [{:keys [blocks-to-add blocks-to-remove]} (get-direct-blocks-and-pages repo tx-report)]
+    ;; update page title indice
+    (let [pages-to-add (filter :block/name blocks-to-add)
+          pages-to-remove (filter :block/name blocks-to-remove)]
+      (when (or (seq pages-to-add) (seq pages-to-remove))
+        (swap! indices update-in [repo :pages]
+               (fn [indice]
+                 (when indice
+                   (doseq [page-entity pages-to-remove]
+                     (.remove indice
+                              (fn [page]
+                                (= (:block/name page-entity)
+                                   (util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
+                   (doseq [page pages-to-add]
+                     (.add indice (bean/->js (original-page-name->index
+                                              (or (:block/original-name page)
+                                                  (:block/name page))))))
+                   indice)))))
+
+    ;; update block indice
+    (when (or (seq blocks-to-add) (seq blocks-to-remove))
+      (let [blocks-to-add (remove nil? (map #(block->index repo (:db-after tx-report) %) blocks-to-add))
+            blocks-to-remove (set (map (comp str :block/uuid) blocks-to-remove))]
+        (bean/->js
+         {:blocks-to-remove-set blocks-to-remove
+          :blocks-to-add        blocks-to-add})))))
+
+(defn exact-matched?
+  "Check if two strings points toward same search result"
+  [q match]
+  (when (and (string? q) (string? match))
+    (boolean
+     (reduce
+      (fn [coll char]
+        (let [coll' (drop-while #(not= char %) coll)]
+          (if (seq coll')
+            (rest coll')
+            (reduced false))))
+      (seq (util/search-normalize match true))
+      (seq (util/search-normalize q true))))))
+
+(defn page-search
+  "Return a list of page names that match the query"
+  [repo db q limit]
+  (when repo
+    (let [q (util/search-normalize q true)
+          q (fuzzy/clean-str q)
+          q (if (= \# (first q)) (subs q 1) q)]
+      (when-not (string/blank? q)
+        (let [indice (or (get-in @indices [repo :pages])
+                         (build-page-indice repo db))
+              result (->> (.search indice q (clj->js {:limit limit}))
+                          (bean/->clj))]
+          (->> result
+               (util/distinct-by (fn [i] (string/trim (get-in i [:item :name]))))
+               (map
+                (fn [{:keys [item]}]
+                  (:original-name item)))
+               (remove nil?)
+               (map string/trim)
+               (distinct)
+               (filter (fn [original-name]
+                         (exact-matched? q original-name)))
+               bean/->js))))))

src/main/frontend/worker/util.cljs

@@ -0,0 +1,45 @@
+(ns frontend.worker.util
+  "Worker utils"
+  (:require [clojure.string :as string]
+            ["remove-accents" :as removeAccents]
+            [medley.core :as medley]
+            [logseq.graph-parser.util :as gp-util]))
+
+(defn search-normalize
+     "Normalize string for searching (loose)"
+     [s remove-accents?]
+     (when s
+       (let [normalize-str (.normalize (string/lower-case s) "NFKC")]
+         (if remove-accents?
+           (removeAccents normalize-str)
+           normalize-str))))
+
+(defn safe-re-find
+  {:malli/schema [:=> [:cat :any :string] [:or :nil :string [:vector [:maybe :string]]]]}
+  [pattern s]
+  (when-not (string? s)
+       ;; TODO: sentry
+    (js/console.trace))
+  (when (string? s)
+    (re-find pattern s)))
+
+(def uuid-pattern "[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}")
+(defonce exactly-uuid-pattern (re-pattern (str "(?i)^" uuid-pattern "$")))
+
+(defn uuid-string?
+  {:malli/schema [:=> [:cat :string] :boolean]}
+  [s]
+  (boolean (safe-re-find exactly-uuid-pattern s)))
+
+(def page-name-sanity-lc
+  "Delegate to gp-util to loosely couple app usages to graph-parser"
+  gp-util/page-name-sanity-lc)
+
+(defn safe-page-name-sanity-lc
+     [s]
+     (if (string? s)
+       (page-name-sanity-lc s) s))
+
+(defn distinct-by
+     [f col]
+     (medley/distinct-by f (seq col)))

src/main/logseq/api.cljs

@@ -15,6 +15,7 @@
             [frontend.handler.recent :as recent-handler]
             [frontend.handler.route :as route-handler]
             [frontend.db :as db]
+            [frontend.db.async :as db-async]
             [frontend.db.model :as db-model]
             [frontend.db.query-dsl :as query-dsl]
             [frontend.db.utils :as db-utils]
@@ -137,11 +138,12 @@
 
 (def ^:export get_current_graph_templates
   (fn []
-    (when (state/get-current-repo)
-      (some-> (db-model/get-all-templates)
-              (update-vals db/pull)
-              (sdk-utils/normalize-keyword-for-json)
-              (bean/->js)))))
+    (when-let [repo (state/get-current-repo)]
+      (let [templates (db-async/<get-all-templates repo)]
+        (some-> templates
+                (update-vals db/pull)
+                (sdk-utils/normalize-keyword-for-json)
+                (bean/->js))))))
 
 (def ^:export get_current_graph
   (fn []
@@ -967,20 +969,21 @@
 
 (defn ^:export insert_template
   [target-uuid template-name]
-  (when-let [target (and (page-handler/template-exists? template-name)
-                         (db-model/get-block-by-uuid target-uuid))]
-    (editor-handler/insert-template! nil template-name {:target target}) nil))
+  (p/let [exists? (page-handler/<template-exists? template-name)]
+    (when exists?
+      (when-let [target (db-model/get-block-by-uuid target-uuid)]
+       (editor-handler/insert-template! nil template-name {:target target}) nil))))
 
 (defn ^:export exist_template
   [name]
-  (page-handler/template-exists? name))
+  (page-handler/<template-exists? name))
 
 (defn ^:export create_template
   [target-uuid template-name ^js opts]
   (when (and template-name (db-model/get-block-by-uuid target-uuid))
-    (let [{:keys [overwrite]} (bean/->clj opts)
-          exist? (page-handler/template-exists? template-name)
-          repo (state/get-current-repo)]
+    (p/let [{:keys [overwrite]} (bean/->clj opts)
+            exist? (page-handler/<template-exists? template-name)
+            repo (state/get-current-repo)]
       (if (or (not exist?) (true? overwrite))
         (do (when-let [old-target (and exist? (db-model/get-template-by-name template-name))]
               (property-handler/remove-block-property! repo (:block/uuid old-target) :template))

src/test/frontend/db/db_based_model_test.cljs

@@ -22,11 +22,6 @@
 
 (use-fixtures :each start-and-destroy-db)
 
-(deftest get-all-properties-test
-  (db-property-handler/set-block-property! repo fbid "property-1" "value" {})
-  (db-property-handler/set-block-property! repo fbid "property-2" "1" {})
-  (is (= '("property-1" "property-2") (model/get-all-properties))))
-
 (deftest get-block-property-values-test
   (db-property-handler/set-block-property! repo fbid "property-1" "value 1" {})
   (db-property-handler/set-block-property! repo sbid "property-1" "value 2" {})
@@ -34,21 +29,21 @@
     (is (= (map second (model/get-block-property-values (:block/uuid property)))
            ["value 1" "value 2"]))))
 
-(deftest get-db-property-values-test
-  (db-property-handler/set-block-property! repo fbid "property-1" "1" {})
-  (db-property-handler/set-block-property! repo sbid "property-1" "2" {})
-  (is (= [1 2] (model/get-db-property-values repo "property-1"))))
+;; (deftest get-db-property-values-test
+;;   (db-property-handler/set-block-property! repo fbid "property-1" "1" {})
+;;   (db-property-handler/set-block-property! repo sbid "property-1" "2" {})
+;;   (is (= [1 2] (model/get-db-property-values repo "property-1"))))
 
-(deftest get-db-property-values-test-with-pages
-  (let [opts {:redirect? false :create-first-block? false}
-        _ (page-handler/create! "page1" opts)
-        _ (page-handler/create! "page2" opts)
-        p1id (:block/uuid (db/entity [:block/name "page1"]))
-        p2id (:block/uuid (db/entity [:block/name "page2"]))]
-    (db-property-handler/upsert-property! repo "property-1" {:type :page} {})
-    (db-property-handler/set-block-property! repo fbid "property-1" p1id {})
-    (db-property-handler/set-block-property! repo sbid "property-1" p2id {})
-    (is (= '("[[page1]]" "[[page2]]") (model/get-db-property-values repo "property-1")))))
+;; (deftest get-db-property-values-test-with-pages
+;;   (let [opts {:redirect? false :create-first-block? false}
+;;         _ (page-handler/create! "page1" opts)
+;;         _ (page-handler/create! "page2" opts)
+;;         p1id (:block/uuid (db/entity [:block/name "page1"]))
+;;         p2id (:block/uuid (db/entity [:block/name "page2"]))]
+;;     (db-property-handler/upsert-property! repo "property-1" {:type :page} {})
+;;     (db-property-handler/set-block-property! repo fbid "property-1" p1id {})
+;;     (db-property-handler/set-block-property! repo sbid "property-1" p2id {})
+;;     (is (= '("[[page1]]" "[[page2]]") (model/get-db-property-values repo "property-1")))))
 
 (deftest get-all-classes-test
   (let [opts {:redirect? false :create-first-block? false :class? true}

src/test/frontend/db/model_test.cljs

@@ -163,27 +163,3 @@ foo:: bar"}])
     (is (= ["child 1" "child 2" "child 3"]
            (map :block/content
                 (model/get-block-immediate-children test-helper/test-db (:block/uuid parent)))))))
-
-(deftest get-property-values
-  (load-test-files [{:file/path "pages/Feature.md"
-                     :file/content "type:: [[Class]]"}
-                    {:file/path "pages/Class.md"
-                     :file/content "type:: https://schema.org/Class\npublic:: true"}
-                    {:file/path "pages/DatePicker.md"
-                     :file/content "type:: #Feature, #Command"}
-                    {:file/path "pages/Whiteboard___Tool___Eraser.md"
-                     :file/content "type:: [[Tool]], [[Whiteboard/Object]]"}])
-
-  (let [type-values (set (model/get-property-values :type))
-        public-values (set (model/get-property-values :public))]
-
-    (is (contains? type-values "[[Class]]")
-        "Property value from single page-ref is wrapped in square brackets")
-    (is (= #{} (set/difference #{"[[Tool]]" "[[Whiteboard/Object]]"} type-values))
-        "Property values from multiple page-refs are wrapped in square brackets")
-    (is (= #{} (set/difference #{"#Feature" "#Command"} type-values))
-        "Property values from multiple tags have hashtags")
-    (is (contains? type-values "https://schema.org/Class")
-        "Property value text is not modified")
-    (is (contains? public-values "true")
-        "Property value that is not text is not modified")))