Add back page content search for file-based graphs

src/main/frontend/components/cmdk.cljs

@@ -26,7 +26,8 @@
     [logseq.graph-parser.util.block-ref :as block-ref]
     [logseq.graph-parser.util :as gp-util]
     [logseq.shui.button.v2 :as button]
-    [frontend.modules.shortcut.utils :as shortcut-utils]))
+    [frontend.modules.shortcut.utils :as shortcut-utils]
+    [frontend.util.text :as text-util]))
 
 (defn translate [t {:keys [id desc]}]
   (when id
@@ -224,6 +225,23 @@
                                       :source-page page)))))]
       (swap! !results update group        merge {:status :success :items items}))))
 
+(defn highlight-content-query
+  "Return hiccup of highlighted content FTS result"
+  [content q]
+  (when-not (or (string/blank? content) (string/blank? q))
+    [:div (loop [content content ;; why recur? because there might be multiple matches
+                 result  []]
+            (let [[b-cut hl-cut e-cut] (text-util/cut-by content "$pfts_2lqh>$" "$<pfts_2lqh$")
+                  hiccups-add [(when-not (string/blank? b-cut)
+                                 [:span b-cut])
+                               (when-not (string/blank? hl-cut)
+                                 [:mark.p-0.rounded-none hl-cut])]
+                  hiccups-add (remove nil? hiccups-add)
+                  new-result (concat result hiccups-add)]
+              (if-not (string/blank? e-cut)
+                (recur e-cut new-result)
+                new-result)))]))
+
 ;; The blocks search action uses an existing handler
 (defmethod load-results :blocks [group state]
   (let [!input (::input state)
@@ -240,7 +258,7 @@
                          (let [id (:block/uuid block)]
                            {:icon "block"
                             :icon-theme :gray
-                            :text (:block/content block)
+                            :text (highlight-content-query (:block/content block) @!input)
                             :header (block/breadcrumb {:search? true} repo id {})
                             :current-page? (when-let [page-id (:block/page block)]
                                              (= page-id (:block/uuid current-page)))

src/main/frontend/regex.cljc

@@ -1,10 +0,0 @@
-(ns ^:no-doc frontend.regex
-  (:require [clojure.string :as string]))
-
-(def re-specials #"([\-\/\\\^\$\*\+\?\.\(\)\|\[\]\{\}])")
-
-(defn escape
-  "Escapes regex characters in string, e.g., 'C++' -> 'C\\+\\+'."
-  [s]
-  ; NOTE: In Clojure, the replace string should be "\\\\$1".
-  (string/replace s re-specials "\\$1"))

src/main/frontend/search.cljs

@@ -6,7 +6,6 @@
             [logseq.graph-parser.config :as gp-config]
             [frontend.db :as db]
             [frontend.db.model :as db-model]
-            [frontend.regex :as regex]
             [frontend.search.agency :as search-agency]
             [frontend.search.db :as search-db :refer [indices]]
             [frontend.search.protocol :as protocol]
@@ -43,8 +42,6 @@
   [s]
   (string/replace (string/lower-case s) #"[\[ \\/_\]\(\)]+" ""))
 
-(def escape-str regex/escape)
-
 (defn char-array
   [s]
   (bean/->js (seq s)))

src/main/frontend/search/db.cljs

@@ -58,25 +58,30 @@
 
 (defn block->index
   "Convert a block to the index for searching"
-  [{:block/keys [uuid page content properties format]
-    :or {format :markdown}}]
-  (let [repo (state/get-current-repo)]
-    (when-not (> (count content) (max-len))
-      (when-not (and (string/blank? content)
-                     (empty? properties))
-        (let [db-based? (config/db-based-graph? repo)
-              content (if db-based? content
-                          (property-util/remove-built-in-properties format content))
-              m {:id (str uuid)
-                 :page (str (:block/uuid page))
-                 :content (sanitize content)}
-              m' (cond-> m
-                   (and db-based? (seq properties))
-                   (update :content
-                           (fn [content]
-                             (str content "\n"
-                                  (get-db-properties-str properties)))))]
-          m')))))
+  [{:block/keys [name uuid page content properties format]
+    :or {format :markdown}
+    :as block}]
+  (let [repo (state/get-current-repo)
+        page? (some? name)
+        block? (nil? name)
+        db-based? (config/db-based-graph? repo)]
+    (when-not (and block? (> (count content) (max-len)))
+      (when-not (and (empty? properties)
+                     (or (and block? (string/blank? content))
+                         (and db-based? page?)))        ; empty page or block
+        (let [content (if block?
+                        (if db-based? content (property-util/remove-built-in-properties format content))
+                        ;; File based page content
+                        (if db-based?
+                          ""            ; empty page content
+                          (some-> (:block/file (db/entity (:db/id block))) :file/content)))
+              content' (if (and db-based? (seq properties))
+                         (str content (when (not= content "") "\n") (get-db-properties-str properties))
+                         content)]
+          (when-not (string/blank? content')
+            {:id (str uuid)
+             :page (str (:block/uuid page))
+             :content (sanitize content')}))))))
 
 (defn original-page-name->index
   [p]

src/main/frontend/worker/search.cljs

@@ -81,6 +81,21 @@
   (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))]
     (.exec db sql)))
 
+(defonce max-snippet-length 250)
+
+(defn- snippet-by
+  [content length]
+  (str (subs content 0 length) (when (> (count content) max-snippet-length) "...")))
+
+(defn- get-snippet-result
+  [snippet]
+  (let [;; Cut snippet to limited size chars for non-matched results
+        flag-highlight "$pfts_2lqh>$ "
+        snippet (if (string/includes? snippet flag-highlight)
+                  snippet
+                  (snippet-by snippet max-snippet-length))]
+    snippet))
+
 (defn- search-blocks-aux
   [db sql input page limit]
   (try
@@ -90,13 +105,15 @@
                                     :rowMode "array"})
                      (.exec db #js {:sql sql
                                     :bind #js [input limit]
-                                    :rowMode "array"}))]
-      (bean/->clj result))
+                                    :rowMode "array"}))
+            blocks (bean/->clj result)]
+      (map (fn [block]
+             (update block 3 get-snippet-result)) blocks))
     (catch :default e
       (prn :debug "Search blocks failed: ")
       (js/console.error e))))
 
-(defn- get-match-inputs
+(defn- get-match-input
   [q]
   (let [match-input (-> q
                         (string/replace " and " " AND ")
@@ -105,9 +122,8 @@
                         (string/replace " | " " OR ")
                         (string/replace " not " " NOT "))]
     (if (not= q match-input)
-      [(string/replace match-input "," "")]
-      [q
-       (str "\"" match-input "\"")])))
+      (string/replace match-input "," "")
+      (str "\"" match-input "\""))))
 
 (defn distinct-by
   [f col]
@@ -117,10 +133,15 @@
   ":page - the page to specifically search on"
   [db q {:keys [limit page]}]
   (when-not (string/blank? q)
-    (p/let [match-inputs (get-match-inputs q)
+    (p/let [match-input (get-match-input q)
             non-match-input (str "%" (string/replace q #"\s+" "%") "%")
             limit  (or limit 20)
-            select "select id, content, page from blocks_fts where "
+            ;; https://www.sqlite.org/fts5.html#the_highlight_function
+            ;; the 2nd column in blocks_fts (content)
+            ;; pfts_2lqh is a key for retrieval
+            ;; highlight and snippet only works for some matching with high rank
+            snippet-aux "snippet(blocks_fts, 1, ' $pfts_2lqh>$ ', ' $<pfts_2lqh$ ', '...', 32)"
+            select (str "select id, page, content, " snippet-aux " from blocks_fts where ")
             pg-sql (if page "page = ? and" "")
             match-sql (str select
                            pg-sql
@@ -128,16 +149,12 @@
             non-match-sql (str select
                                pg-sql
                                " content like ? limit ?")
-            results (p/all (map
-                            (fn [match-input]
-                              (search-blocks-aux db match-sql match-input page limit))
-                            match-inputs))
-            matched-result (apply concat results)
+            matched-result (search-blocks-aux db match-sql match-input page limit)
             non-match-result (search-blocks-aux db non-match-sql non-match-input page limit)
             all-result (->> (concat matched-result non-match-result)
-                            (map (fn [[id content page]]
+                            (map (fn [[id _content page snippet]]
                                    {:uuid id
-                                    :content content
+                                    :content snippet
                                     :page page})))]
       (->>
        all-result