Merge branch 'cnrpman-page-level-fts'

e2e-tests/page-search.spec.ts

@@ -37,11 +37,12 @@ import { IsMac, createRandomPage, newBlock, newInnerBlock, randomString, lastBlo
   await page.waitForSelector('[placeholder="Search or create page"]')
   await page.fill('[placeholder="Search or create page"]', 'Einführung in die Allgemeine Sprachwissenschaft' + rand)
 
-  await page.waitForTimeout(500)
+  await page.waitForTimeout(2000) // wait longer for search contents to render
   const results = await page.$$('#ui__ac-inner>div')
-  expect(results.length).toEqual(3) // 2 blocks + 1 page
+  expect(results.length).toBeGreaterThan(3) // 2 blocks + 1 page + 2 page content
   await page.keyboard.press("Escape")
   await page.keyboard.press("Escape")
+  await page.waitForTimeout(1000) // wait for modal disappear
 })
 
 async function alias_test(page: Page, page_name: string, search_kws: string[]) {

src/electron/electron/handler.cljs

@@ -285,28 +285,47 @@
   (async/put! state/persistent-dbs-chan true)
   true)
 
+;; Search related IPCs
 (defmethod handle :search-blocks [_window [_ repo q opts]]
   (search/search-blocks repo q opts))
 
-(defmethod handle :rebuild-blocks-indice [_window [_ repo data]]
+(defmethod handle :search-pages [_window [_ repo q opts]]
+  (search/search-pages repo q opts))
+
+(defmethod handle :rebuild-indice [_window [_ repo block-data page-data]]
   (search/truncate-blocks-table! repo)
   ;; unneeded serialization
-  (search/upsert-blocks! repo (bean/->js data))
+  (search/upsert-blocks! repo (bean/->js block-data))
+  (search/truncate-pages-table! repo)
+  (search/upsert-pages! repo (bean/->js page-data))
   [])
 
 (defmethod handle :transact-blocks [_window [_ repo data]]
   (let [{:keys [blocks-to-remove-set blocks-to-add]} data]
+    ;; Order matters! Same id will delete then upsert sometimes.
     (when (seq blocks-to-remove-set)
       (search/delete-blocks! repo blocks-to-remove-set))
     (when (seq blocks-to-add)
       ;; unneeded serialization
       (search/upsert-blocks! repo (bean/->js blocks-to-add)))))
 
-(defmethod handle :truncate-blocks [_window [_ repo]]
-  (search/truncate-blocks-table! repo))
+(defmethod handle :transact-pages [_window [_ repo data]]
+  (let [{:keys [pages-to-remove-set pages-to-add]} data]
+    ;; Order matters! Same id will delete then upsert sometimes.
+    (when (seq pages-to-remove-set)
+      (search/delete-pages! repo pages-to-remove-set))
+    (when (seq pages-to-add)
+      ;; unneeded serialization
+      (search/upsert-pages! repo (bean/->js pages-to-add)))))
+
+(defmethod handle :truncate-indice [_window [_ repo]]
+  (search/truncate-blocks-table! repo)
+  (search/truncate-pages-table! repo))
 
 (defmethod handle :remove-db [_window [_ repo]]
   (search/delete-db! repo))
+;; ^^^^
+;; Search related IPCs End
 
 (defn clear-cache!
   [window]

src/electron/electron/search.cljs

@@ -1,4 +1,5 @@
 (ns electron.search
+  "Provides both page level and block level index"
   (:require ["path" :as path]
             ["fs-extra" :as fs]
             ["better-sqlite3" :as sqlite3]
@@ -31,25 +32,52 @@
   (when db
     (.prepare db sql)))
 
-(defn add-triggers!
+(defn add-blocks-fts-triggers!
+  "Table bindings of blocks tables and the blocks FTS virtual tables"
   [db]
-  (let [triggers ["CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks
-    BEGIN
-        DELETE from blocks_fts where rowid = old.id;
-    END;"
+  (let [triggers [;; add
+                  "CREATE TRIGGER IF NOT EXISTS blocks_ad AFTER DELETE ON blocks
+                  BEGIN
+                      DELETE from blocks_fts where rowid = old.id;
+                  END;"
+                  ;; insert
                   "CREATE TRIGGER IF NOT EXISTS blocks_ai AFTER INSERT ON blocks
-    BEGIN
-        INSERT INTO blocks_fts (rowid, uuid, content, page)
-        VALUES (new.id, new.uuid, new.content, new.page);
-    END;
-"
+                  BEGIN
+                      INSERT INTO blocks_fts (rowid, uuid, content, page)
+                      VALUES (new.id, new.uuid, new.content, new.page);
+                  END;"
+                  ;; update
                   "CREATE TRIGGER IF NOT EXISTS blocks_au AFTER UPDATE ON blocks
-    BEGIN
-        DELETE from blocks_fts where rowid = old.id;
-        INSERT INTO blocks_fts (rowid, uuid, content, page)
-        VALUES (new.id, new.uuid, new.content, new.page);
-    END;"
-                  ]]
+                  BEGIN
+                      DELETE from blocks_fts where rowid = old.id;
+                      INSERT INTO blocks_fts (rowid, uuid, content, page)
+                      VALUES (new.id, new.uuid, new.content, new.page);
+                  END;"]]
+    (doseq [trigger triggers]
+      (let [stmt (prepare db trigger)]
+        (.run ^object stmt)))))
+
+(defn add-pages-fts-triggers!
+  "Table bindings of pages tables and the pages FTS virtual tables"
+  [db]
+  (let [triggers [;; add
+                  "CREATE TRIGGER IF NOT EXISTS pages_ad AFTER DELETE ON pages
+                  BEGIN
+                      DELETE from pages_fts where rowid = old.id;
+                  END;"
+                  ;; insert
+                  "CREATE TRIGGER IF NOT EXISTS pages_ai AFTER INSERT ON pages
+                  BEGIN
+                      INSERT INTO pages_fts (rowid, uuid, content)
+                      VALUES (new.id, new.uuid, new.content);
+                  END;"
+                  ;; update
+                  "CREATE TRIGGER IF NOT EXISTS pages_au AFTER UPDATE ON pages
+                  BEGIN
+                      DELETE from pages_fts where rowid = old.id;
+                      INSERT INTO pages_fts (rowid, uuid, content)
+                      VALUES (new.id, new.uuid, new.content);
+                  END;"]]
     (doseq [trigger triggers]
       (let [stmt (prepare db trigger)]
         (.run ^object stmt)))))
@@ -68,6 +96,19 @@
   (let [stmt (prepare db "CREATE VIRTUAL TABLE IF NOT EXISTS blocks_fts USING fts5(uuid, content, page)")]
     (.run ^object stmt)))
 
+(defn create-pages-table!
+  [db]
+  (let [stmt (prepare db "CREATE TABLE IF NOT EXISTS pages (
+                        id INTEGER PRIMARY KEY,
+                        uuid TEXT NOT NULL,
+                        content TEXT NOT NULL)")]
+    (.run ^object stmt)))
+
+(defn create-pages-fts-table!
+  [db]
+  (let [stmt (prepare db "CREATE VIRTUAL TABLE IF NOT EXISTS pages_fts USING fts5(uuid, content)")]
+    (.run ^object stmt)))
+
 (defn get-search-dir
   []
   (let [path (.getPath ^object app "userData")]
@@ -96,7 +137,10 @@
       (try (let [db (sqlite3 db-full-path nil)]
              (create-blocks-table! db)
              (create-blocks-fts-table! db)
-             (add-triggers! db)
+             (create-pages-table! db)
+             (create-pages-fts-table! db)
+             (add-blocks-fts-triggers! db)
+             (add-pages-fts-triggers! db)
              (swap! databases assoc db-sanitized-name db))
            (catch :default e
              (logger/error (str e ": " db-name))
@@ -111,6 +155,36 @@
       (doseq [db-name dbs]
         (open-db! db-name)))))
 
+(defn- clj-list->sql
+  "Turn clojure list into SQL list
+   '(1 2 3 4)
+   ->
+   \"('1','2','3','4')\""
+  [ids]
+  (str "(" (->> (map (fn [id] (str "'" id "'")) ids)
+                (string/join ", ")) ")"))
+
+(defn upsert-pages!
+  [repo pages]
+  (if-let [db (get-db repo)]
+    ;; TODO: what if a CONFLICT on uuid
+    (let [insert (prepare db "INSERT INTO pages (id, uuid, content) VALUES (@id, @uuid, @content) ON CONFLICT (id) DO UPDATE SET content = @content")
+          insert-many (.transaction ^object db
+                                    (fn [pages]
+                                      (doseq [page pages]
+                                        (.run ^object insert page))))]
+      (insert-many pages))
+    (do
+      (open-db! repo)
+      (upsert-pages! repo pages))))
+
+(defn delete-pages!
+  [repo ids]
+  (when-let [db (get-db repo)]
+    (let [sql (str "DELETE from pages WHERE id IN " (clj-list->sql ids))
+          stmt (prepare db sql)]
+      (.run ^object stmt))))
+
 (defn upsert-blocks!
   [repo blocks]
   (if-let [db (get-db repo)]
@@ -128,9 +202,7 @@
 (defn delete-blocks!
   [repo ids]
   (when-let [db (get-db repo)]
-    (let [ids (->> (map (fn [id] (str "'" id "'")) ids)
-                   (string/join ", "))
-          sql (str "DELETE from blocks WHERE id IN (" ids ")")
+    (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))
           stmt (prepare db sql)]
       (.run ^object stmt))))
 
@@ -150,19 +222,35 @@
        (.all ^object stmt  input limit))
      :keywordize-keys true)))
 
+(defn- get-match-inputs
+  [q]
+  (let [match-input (-> q
+                        (string/replace " and " " AND ")
+                        (string/replace " & " " AND ")
+                        (string/replace " or " " OR ")
+                        (string/replace " | " " OR ")
+                        (string/replace " not " " NOT "))]
+    (if (not= q match-input)
+      [(string/replace match-input "," "")]
+      [q
+       (str "\"" match-input "\"")])))
+
+(defn distinct-by
+  [f col]
+  (reduce
+   (fn [acc x]
+     (if (some #(= (f x) (f %)) acc)
+       acc
+       (vec (conj acc x))))
+   []
+   col))
+
 (defn search-blocks
+  ":page - the page to specificly search on"
   [repo q {:keys [limit page]}]
   (when-let [database (get-db repo)]
     (when-not (string/blank? q)
-      (let [match-input (-> q
-                            (string/replace " and " " AND ")
-                            (string/replace " & " " AND ")
-                            (string/replace " or " " OR ")
-                            (string/replace " | " " OR ")
-                            (string/replace " not " " NOT "))
-            match-input (if (not= q match-input)
-                          (string/replace match-input "," "")
-                          (str "\"" match-input "\""))
+      (let [match-inputs (get-match-inputs q)
             non-match-input (str "%" (string/replace q #"\s+" "%") "%")
             limit  (or limit 20)
             select "select rowid, uuid, content, page from blocks_fts where "
@@ -172,12 +260,62 @@
                            " content match ? order by rank limit ?")
             non-match-sql (str select
                                pg-sql
-                               " content like ? limit ?")]
+                               " content like ? limit ?")
+            matched-result (->>
+                            (map
+                              (fn [match-input]
+                                (search-blocks-aux database match-sql match-input page limit))
+                              match-inputs)
+                            (apply concat))]
+        (->>
+         (concat matched-result
+                 (search-blocks-aux database non-match-sql non-match-input page limit))
+         (distinct-by :id)
+         (take limit)
+         (vec))))))
+
+(defn- search-pages-res-unpack
+  [arr]
+  (let [[rowid uuid content snippet] arr]
+    {:id      rowid
+     :uuid    uuid
+     :content content
+     :snippet snippet}))
+
+(defn- search-pages-aux
+  [database sql input limit]
+  (let [stmt (prepare database sql)]
+    (map search-pages-res-unpack (-> (.raw ^object stmt)
+                                     (.all input limit)
+                                     (js->clj)))))
+
+(defn search-pages
+  [repo q {:keys [limit]}]
+  (when-let [database (get-db repo)]
+    (when-not (string/blank? q)
+      (let [match-inputs (get-match-inputs q)
+            non-match-input (str "%" (string/replace q #"\s+" "%") "%")
+            limit  (or limit 20)
+            ;; https://www.sqlite.org/fts5.html#the_highlight_function
+            ;; the 2nd column in pages_fts (content)
+            ;; pfts_2lqh is a key for retrieval
+            ;; highlight and snippet only works for some matching with high rank
+            snippet-aux "snippet(pages_fts, 1, '$pfts_2lqh>$', '$<pfts_2lqh$', '...', 32)"
+            select (str "select rowid, uuid, content, " snippet-aux " from pages_fts where ")
+            match-sql (str select
+                           " content match ? order by rank limit ?")
+            non-match-sql (str select
+                               " content like ? limit ?")
+            matched-result (->>
+                            (map
+                              (fn [match-input]
+                                (search-pages-aux database match-sql match-input limit))
+                              match-inputs)
+                            (apply concat))]
         (->>
-         (concat
-          (search-blocks-aux database match-sql match-input page limit)
-          (search-blocks-aux database non-match-sql non-match-input page limit))
-         (distinct)
+         (concat matched-result
+          (search-pages-aux database non-match-sql non-match-input limit))
+         (distinct-by :id)
          (take limit)
          (vec))))))
 
@@ -191,6 +329,16 @@
                         "delete from blocks_fts;")]
       (.run ^object stmt))))
 
+(defn truncate-pages-table!
+  [repo]
+  (when-let [database (get-db repo)]
+    (let [stmt (prepare database
+                        "delete from pages;")
+          _ (.run ^object stmt)
+          stmt (prepare database
+                        "delete from pages_fts;")]
+      (.run ^object stmt))))
+
 (defn delete-db!
   [repo]
   (when-let [database (get-db repo)]
@@ -205,9 +353,3 @@
   (when-let [database (get-db repo)]
     (let [stmt (prepare database sql)]
       (.all ^object stmt))))
-
-(comment
-  (def repo (first (keys @databases)))
-  (query repo
-         "select * from blocks_fts")
-  (delete-db! repo))

src/main/frontend/components/block.cljs

@@ -2497,12 +2497,14 @@
 (rum/defc breadcrumb-separator [] [:span.mx-2.opacity-50 "➤"])
 
 (defn breadcrumb
+  "block-id - uuid of the target block of breadcrumb. page uuid is also acceptable"
   [config repo block-id {:keys [show-page? indent? end-separator? level-limit _navigating-block]
                          :or {show-page? true
                               level-limit 3}
                          :as opts}]
   (let [parents (db/get-block-parents repo block-id (inc level-limit))
-        page (db/get-block-page repo block-id)
+        page (or (db/get-block-page repo block-id) ;; only return for block uuid
+                 (model/query-block-by-uuid block-id)) ;; return page entity when received page uuid
         page-name (:block/name page)
         page-original-name (:block/original-name page)
         show? (or (seq parents) show-page? page-name)

src/main/frontend/components/search.cljs

@@ -22,7 +22,8 @@
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
             [reitit.frontend.easy :as rfe]
-            [frontend.modules.shortcut.core :as shortcut]))
+            [frontend.modules.shortcut.core :as shortcut]
+            [frontend.util.text :as text-util]))
 
 (defn highlight-exact-query
   [content q]
@@ -62,12 +63,43 @@
                              (conj result [:span content])))]
             [:p {:class "m-0"} elements]))))))
 
+(defn highlight-page-content-query
+  "Return hiccup of highlighted page content FTS result"
+  [content q]
+  (when-not (or (string/blank? content) (string/blank? q))
+    [:div (loop [content content ;; why recur? because there might be multiple matches
+                 result  []]
+            (let [[b-cut hl-cut e-cut] (text-util/cut-by content "$pfts_2lqh>$" "$<pfts_2lqh$")
+                  hiccups-add [(when-not (string/blank? b-cut)
+                                 [:span b-cut])
+                               (when-not (string/blank? hl-cut)
+                                 [:mark.p-0.rounded-none hl-cut])]
+                  hiccups-add (remove nil? hiccups-add)
+                  new-result (concat result hiccups-add)]
+              (if-not (string/blank? e-cut)
+                (recur e-cut new-result)
+                new-result)))]))
+
 (rum/defc search-result-item
   [icon content]
   [:.search-result
    (ui/type-icon icon)
    [:.self-center content]])
 
+(rum/defc page-content-search-result-item
+  [repo uuid format snippet q search-mode]
+  [:div
+   (when (not= search-mode :page)
+     [:div {:class "mb-1" :key "parents"}
+      (block/breadcrumb {:id "block-search-block-parent"
+                         :block? true
+                         :search? true}
+                        repo
+                        (clojure.core/uuid uuid)
+                        {:indent? false})])
+   [:div {:class "font-medium" :key "content"}
+    (highlight-page-content-query (search-handler/sanity-search-content format snippet) q)]])
+
 (rum/defc block-search-result-item
   [repo uuid format content q search-mode]
   (let [content (search-handler/sanity-search-content format content)]
@@ -157,6 +189,21 @@
         (println "[Error] Block page missing: "
                  {:block-id block-uuid
                   :block (db/pull [:block/uuid block-uuid])})))
+
+    :page-content
+    (let [page-uuid (uuid (:block/uuid data))
+          page (model/get-block-by-uuid page-uuid)
+          page-name (:block/name page)]
+      (if page
+        (cond
+          (model/whiteboard-page? page-name)
+          (route/redirect-to-whiteboard! page-name)
+          :else
+          (route/redirect-to-page! page-name))
+        ;; search indice outdated
+        (println "[Error] page missing: "
+                 {:page-uuid page-uuid
+                  :page page})))
     nil)
   (state/close-modal!))
 
@@ -172,6 +219,19 @@
          repo
          (:db/id page)
          :page)))
+    
+    :page-content
+    (let [page-uuid (uuid (:block/uuid data))
+          page (model/get-block-by-uuid page-uuid)]
+      (if page
+        (state/sidebar-add-block!
+         repo
+         (:db/id page)
+         :page)
+        ;; search indice outdated
+        (println "[Error] page missing: "
+                 {:page-uuid page-uuid
+                  :page page})))
 
     :block
     (let [block-uuid (uuid (:block/uuid data))
@@ -254,10 +314,24 @@
                                 (do (log/error "search result with non-existing uuid: " data)
                                     (str "Cache is outdated. Please click the 'Re-index' button in the graph's dropdown menu."))))])
 
+       :page-content
+       (let [{:block/keys [snippet uuid]} data  ;; content here is normalized
+             repo (state/sub :git/current-repo)
+             page (model/query-block-by-uuid uuid)  ;; it's actually a page
+             format (db/get-page-format page)]
+         [:span {:data-block-ref uuid}
+          (search-result-item {:name "page"
+                               :title (t :search-item/page)
+                               :extension? true}
+                              (if page
+                                (page-content-search-result-item repo uuid format snippet search-q search-mode)
+                                (do (log/error "search result with non-existing uuid: " data)
+                                    (str "Cache is outdated. Please click the 'Re-index' button in the graph's dropdown menu."))))])
+
        nil)]))
 
 (rum/defc search-auto-complete
-  [{:keys [engine pages files blocks has-more?] :as result} search-q all?]
+  [{:keys [engine pages files pages-content blocks has-more?] :as result} search-q all?]
   (let [pages (when-not all? (map (fn [page]
                                     (let [alias (model/get-redirect-page-name page)]
                                       (cond->
@@ -270,6 +344,7 @@
                                   (remove nil? pages)))
         files (when-not all? (map (fn [file] {:type :file :data file}) files))
         blocks (map (fn [block] {:type :block :data block}) blocks)
+        pages-content (map (fn [pages-content] {:type :page-content :data pages-content}) pages-content)
         search-mode (state/sub :search/mode)
         new-page (if (or
                       (some? engine)
@@ -284,13 +359,13 @@
                      [{:type :new-page}]))
         result (cond
                  config/publishing?
-                 (concat pages files blocks)
+                 (concat pages files blocks) ;; Browser doesn't have page content FTS
 
                  (= :whiteboard/link search-mode)
-                 (concat pages blocks)
+                 (concat pages blocks pages-content)
 
                  :else
-                 (concat new-page pages files blocks))
+                 (concat new-page pages files blocks pages-content))
         result (if (= search-mode :graph)
                  [{:type :graph-add-filter}]
                  result)

src/main/frontend/db/model.cljs

@@ -263,6 +263,7 @@
   (db-utils/entity [:block/uuid (if (uuid? id) id (uuid id))]))
 
 (defn query-block-by-uuid
+  "Return block or page entity, depends on the uuid"
   [id]
   (db-utils/pull [:block/uuid (if (uuid? id) id (uuid id))]))
 
@@ -774,6 +775,8 @@
         react)))))
 
 (defn get-page-blocks-no-cache
+  "Return blocks of the designated page, without using cache.
+   page - name / title of the page"
   ([page]
    (get-page-blocks-no-cache (state/get-current-repo) page nil))
   ([repo-url page]
@@ -1517,6 +1520,7 @@
             assets (get-assets datoms)]
         [@(d/conn-from-datoms datoms db-schema/schema) assets]))))
 
+;; Deprecated?
 (defn delete-blocks
   [repo-url files _delete-page?]
   (when (seq files)

src/main/frontend/handler/search.cljs

@@ -44,12 +44,14 @@
                         (:db/id (db/entity repo [:block/name (util/page-name-sanity-lc page-db-id)]))
                         page-db-id)
            opts (if page-db-id (assoc opts :page (str page-db-id)) opts)]
-       (p/let [blocks (search/block-search repo q opts)]
+       (p/let [blocks (search/block-search repo q opts)
+               pages-content (search/page-content-search repo q opts)]
          (let [result (merge
                        {:blocks blocks
                         :has-more? (= limit (count blocks))}
                        (when-not page-db-id
-                         {:pages (search/page-search q)
+                         {:pages-content pages-content
+                          :pages (search/page-search q)
                           :files (search/file-search q)}))
                search-key (if more? :search/more-result :search/result)]
            (swap! state/state assoc search-key result)

src/main/frontend/modules/datascript_report/core.cljs

@@ -13,6 +13,8 @@
       nil)))
 
 (defn get-entity-from-db-after-or-before
+  "Get the entity from db after if possible; otherwise get entity from db before
+   Useful for fetching deleted elements"
   [db-before db-after db-id]
   (let [r (safe-pull db-after '[*] db-id)]
     (if (= keys-of-deleted-entity (count r))
@@ -21,6 +23,7 @@
       r)))
 
 (defn get-blocks-and-pages
+  "Calculate updated blocks and pages based on the db-before and db-after from tx-report"
   [{:keys [db-before db-after tx-data tx-meta]}]
   (let [updated-db-ids (-> (mapv first tx-data) (set))
         result (reduce
@@ -39,6 +42,7 @@
                 {:blocks #{}
                  :pages #{}}
                 updated-db-ids)
+        ;; updated pages logged in tx-meta (usually from move op)
         tx-meta-pages (->> [(:from-page tx-meta) (:target-page tx-meta)]
                            (remove nil?)
                            (map #(get-entity-from-db-after-or-before db-before db-after %))

src/main/frontend/search.cljs

@@ -14,7 +14,9 @@
             [frontend.util :as util]
             [frontend.util.property :as property]
             [goog.object :as gobj]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [clojure.set :as set]
+            [frontend.modules.datascript-report.core :as db-report]))
 
 (defn get-engine
   [repo]
@@ -95,11 +97,24 @@
       (when-not (string/blank? q)
         (protocol/query engine q option)))))
 
+(defn page-content-search
+  [repo q option]
+  (when-let [engine (get-engine repo)]
+    (let [q (util/search-normalize q (state/enable-search-remove-accents?))
+          q (if (util/electron?) q (escape-str q))]
+      (when-not (string/blank? q)
+        (protocol/query-page engine q option)))))
+
 (defn- transact-blocks!
   [repo data]
   (when-let [engine (get-engine repo)]
     (protocol/transact-blocks! engine data)))
 
+(defn- transact-pages!
+  [repo data] 
+  (when-let [engine (get-engine repo)]
+    (protocol/transact-pages! engine data)))
+
 (defn exact-matched?
   "Check if two strings points toward same search result"
   [q match]
@@ -124,7 +139,7 @@
            q (clean-str q)]
        (when-not (string/blank? q)
          (let [indice (or (get-in @indices [repo :pages])
-                          (search-db/make-pages-indice!))
+                          (search-db/make-pages-title-indice!))
                result (->> (.search indice q (clj->js {:limit limit}))
                            (bean/->clj))]
            ;; TODO: add indexes for highlights
@@ -191,8 +206,48 @@
            (let [result (fuzzy-search result q :limit limit)]
              (vec result))))))))
 
-(defn sync-search-indice!
-  [repo tx-report]
+(defn- get-pages-from-datoms-impl
+  [pages]
+  (let [pages-result (db/pull-many '[:db/id :block/name :block/original-name] (set (map :e pages)))
+        pages-to-add-set (->> (filter :added pages)
+                              (map :e)
+                              (set))
+        pages-to-add (->> (filter (fn [page]
+                                    (contains? pages-to-add-set (:db/id page))) pages-result)
+                          (map (fn [p] (or (:block/original-name p)
+                                           (:block/name p))))
+                          (map search-db/original-page-name->index))
+        pages-to-remove-set (->> (remove :added pages)
+                                 (map :v))
+        pages-to-remove-id-set (->> (remove :added pages)
+                                    (map :e)
+                                    set)]
+    {:pages-to-add        pages-to-add
+     :pages-to-remove-set pages-to-remove-set
+     :pages-to-add-id-set pages-to-add-set
+     :pages-to-remove-id-set pages-to-remove-id-set}))
+
+(defn- get-blocks-from-datoms-impl
+  [blocks]
+  (when (seq blocks)
+    (let [blocks-result (->> (db/pull-many '[:db/id :block/uuid :block/format :block/content :block/page] (set (map :e blocks)))
+                             (map (fn [b] (assoc b :block/page (get-in b [:block/page :db/id])))))
+          blocks-to-add-set (->> (filter :added blocks)
+                                 (map :e)
+                                 (set))
+          blocks-to-add (->> (filter (fn [block]
+                                       (contains? blocks-to-add-set (:db/id block)))
+                                     blocks-result)
+                             (map search-db/block->index)
+                             (remove nil?))
+          blocks-to-remove-set (->> (remove :added blocks)
+                                    (map :e)
+                                    (set))]
+      {:blocks-to-remove-set blocks-to-remove-set
+       :blocks-to-add        blocks-to-add})))
+
+(defn- get-direct-blocks-and-pages 
+  [tx-report]
   (let [data (:tx-data tx-report)
         datoms (filter
                 (fn [datom]
@@ -200,50 +255,78 @@
                 data)]
     (when (seq datoms)
       (let [datoms (group-by :a datoms)
-            pages (:block/name datoms)
-            blocks (:block/content datoms)]
-        (when (seq pages)
-          (let [pages-result (db/pull-many '[:db/id :block/name :block/original-name] (set (map :e pages)))
-                pages-to-add-set (->> (filter :added pages)
-                                      (map :e)
-                                      (set))
-                pages-to-add (->> (filter (fn [page]
-                                            (contains? pages-to-add-set (:db/id page))) pages-result)
-                                  (map (fn [p] (or (:block/original-name p)
-                                                   (:block/name p))))
-                                  (map search-db/original-page-name->index))
-                pages-to-remove-set (->> (remove :added pages)
-                                         (map :v))]
-            (swap! search-db/indices update-in [repo :pages]
-                   (fn [indice]
-                     (when indice
-                       (doseq [page-name pages-to-remove-set]
-                         (.remove indice
-                                  (fn [page]
-                                    (= (util/safe-page-name-sanity-lc page-name)
-                                       (util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
-                       (when (seq pages-to-add)
-                         (doseq [page pages-to-add]
-                           (.add indice (bean/->js page)))))
-                     indice))))
+            blocks (:block/content datoms)
+            pages (:block/name datoms)]
+        (merge (get-blocks-from-datoms-impl blocks)
+               (get-pages-from-datoms-impl pages))))))
+
+(defn- get-indirect-pages
+  "Return the set of pages that will have content updated"
+  [tx-report]
+  (let [data   (:tx-data tx-report)
+        datoms (filter
+                (fn [datom]
+                  (and (:added datom)
+                       (contains? #{:file/content} (:a datom))))
+                data)]
+    (when (seq datoms)
+      (->> datoms
+           (mapv (fn [datom]
+                   (let [tar-db  (:db-after tx-report)]
+                     ;; Reverse query the corresponding page id of the modified `:file/content`)
+                     (when-let [page-id (->> (:e datom)
+                                             (db-report/safe-pull tar-db '[:block/_file])
+                                             (:block/_file)
+                                             (first)
+                                             (:db/id))]
+                       ;; Fetch page entity according to what page->index requested
+                       (db-report/safe-pull tar-db '[:db/id :block/uuid
+                                                     :block/original-name
+                                                     {:block/file [:file/content]}]
+                                            page-id)))))
+           (remove nil?)))))
+
+;; TODO merge with logic in `invoke-hooks` when feature and test is sufficient
+(defn sync-search-indice!
+  [repo tx-report]
+  (let [{:keys [pages-to-add pages-to-remove-set pages-to-remove-id-set
+                blocks-to-add blocks-to-remove-set]} (get-direct-blocks-and-pages tx-report) ;; directly modified block & pages
+        updated-pages (get-indirect-pages tx-report)]
+    ;; update page title indice
+    (when (or (seq pages-to-add) (seq pages-to-remove-set))
+      (swap! search-db/indices update-in [repo :pages]
+             (fn [indice]
+               (when indice
+                 (doseq [page-name pages-to-remove-set]
+                   (.remove indice
+                            (fn [page]
+                              (= (util/safe-page-name-sanity-lc page-name)
+                                 (util/safe-page-name-sanity-lc (gobj/get page "original-name"))))))
+                 (when (seq pages-to-add)
+                   (doseq [page pages-to-add]
+                     (.add indice (bean/->js page)))))
+               indice)))
+
+    ;; update block indice
+    (when (or (seq blocks-to-add) (seq blocks-to-remove-set))
+      (transact-blocks! repo
+                        {:blocks-to-remove-set blocks-to-remove-set
+                         :blocks-to-add        blocks-to-add}))
 
-        (when (seq blocks)
-          (let [blocks-result (->> (db/pull-many '[:db/id :block/uuid :block/format :block/content :block/page] (set (map :e blocks)))
-                                   (map (fn [b] (assoc b :block/page (get-in b [:block/page :db/id])))))
-                blocks-to-add-set (->> (filter :added blocks)
-                                       (map :e)
-                                       (set))
-                blocks-to-add (->> (filter (fn [block]
-                                             (contains? blocks-to-add-set (:db/id block)))
-                                           blocks-result)
-                                   (map search-db/block->index)
-                                   (remove nil?))
-                blocks-to-remove-set (->> (remove :added blocks)
-                                          (map :e)
-                                          (set))]
-            (transact-blocks! repo
-                              {:blocks-to-remove-set blocks-to-remove-set
-                               :blocks-to-add blocks-to-add})))))))
+    ;; update page indice
+    (when (or (seq pages-to-remove-id-set) (seq updated-pages)) ;; when move op happens, no :block/content provided
+      (let [indice-pages   (map search-db/page->index updated-pages)
+            invalid-set    (->> (map (fn [updated indiced] ;; get id of pages without valid page index
+                                       (if indiced nil (:db/id updated)))
+                                     updated-pages indice-pages)
+                                (remove nil?)
+                                set)
+            pages-to-add   (->> indice-pages
+                                (remove nil?)
+                                set)
+            pages-to-remove-set (set/union pages-to-remove-id-set invalid-set)]
+        (transact-pages! repo {:pages-to-remove-set pages-to-remove-set
+                               :pages-to-add        pages-to-add})))))
 
 (defn rebuild-indices!
   ([]
@@ -251,10 +334,10 @@
   ([repo]
    (when repo
      (when-let [engine (get-engine repo)]
-       (let [pages (search-db/make-pages-indice!)]
+       (let [page-titles (search-db/make-pages-title-indice!)]
          (p/let [blocks (protocol/rebuild-blocks-indice! engine)]
-           (let [result {:pages pages
-                         :blocks blocks}]
+           (let [result {:pages         page-titles ;; TODO: rename key to :page-titles
+                         :blocks        blocks}]
              (swap! indices assoc repo result)
              indices)))))))
 

src/main/frontend/search/agency.cljs

@@ -31,6 +31,13 @@
         (protocol/query e q opts))
       (protocol/query e1 q opts)))
 
+  (query-page [_this q opts]
+    (println "D:Search > Query-page contents:" repo q opts)
+    (let [[e1 e2] (get-registered-engines repo)]
+      (doseq [e e2]
+        (protocol/query-page e q opts))
+      (protocol/query-page e1 q opts)))
+
   (rebuild-blocks-indice! [_this]
     (println "D:Search > Initial blocks indice!:" repo)
     (let [[e1 e2] (get-registered-engines repo)]
@@ -43,6 +50,11 @@
     (doseq [e (get-flatten-registered-engines repo)]
       (protocol/transact-blocks! e data)))
 
+  (transact-pages! [_this data]
+    (println "D:Search > Transact pages!:" repo)
+    (doseq [e (get-flatten-registered-engines repo)]
+      (protocol/transact-pages! e data)))
+
   (truncate-blocks! [_this]
     (println "D:Search > Truncate blocks!" repo)
     (doseq [e (get-flatten-registered-engines repo)]

src/main/frontend/search/browser.cljs

@@ -35,6 +35,7 @@
   protocol/Engine
   (query [_this q option]
     (p/promise (search-blocks repo q option)))
+  (query-page [_this _q _opt] nil) ;; Page index is not available with fuse.js until sufficient performance benchmarking
   (rebuild-blocks-indice! [_this]
     (let [indice (search-db/make-blocks-indice! repo)]
       (p/promise indice)))
@@ -51,6 +52,7 @@
                  (doseq [block blocks-to-add]
                    (.add indice (bean/->js block)))))
              indice)))
+  (transact-pages! [_this _data] nil) ;; Page index is not available with fuse.js until sufficient performance benchmarking
   (truncate-blocks! [_this]
     (swap! indices assoc-in [repo :blocks] nil))
   (remove-db! [_this]

src/main/frontend/search/db.cljs

@@ -10,15 +10,33 @@
 
 (defonce indices (atom nil))
 
+(defn- sanitize
+  [content]
+  (util/search-normalize content (state/enable-search-remove-accents?)))
+
+(defn- max-len
+  []
+  (state/block-content-max-length (state/get-current-repo)))
+
 (defn block->index
   "Convert a block to the index for searching"
   [{:block/keys [uuid page content] :as block}]
-  (when-let [content (util/search-normalize content (state/enable-search-remove-accents?))]
-    (when-not (> (count content) (state/block-content-max-length (state/get-current-repo)))
-      {:id (:db/id block)
+  (when-not (> (count content) (max-len))
+    {:id (:db/id block)
+     :uuid (str uuid)
+     :page page
+     :content (sanitize content)}))
+
+(defn page->index
+  "Convert a page name to the index for searching (page content level)
+   Generate index based on the DB content AT THE POINT OF TIME"
+  [{:block/keys [uuid _original-name] :as page}]
+  (when-let [content (some-> (:block/file page)
+                             (:file/content))]
+    (when-not (> (count content) (* (max-len) 10))
+      {:id   (:db/id page)
        :uuid (str uuid)
-       :page page
-       :content content})))
+       :content (sanitize content)})))
 
 (defn build-blocks-indice
   ;; TODO: Remove repo effects fns further up the call stack. db fns need standardization on taking connection
@@ -29,6 +47,14 @@
        (remove nil?)
        (bean/->js)))
 
+(defn build-pages-indice 
+  [repo]
+  (->> (db/get-all-pages repo)
+       (map #(db/entity (:db/id %))) ;; get full file-content
+       (map page->index)
+       (remove nil?)
+       (bean/->js)))
+
 (defn make-blocks-indice!
   [repo]
   (let [blocks (build-blocks-indice repo)
@@ -46,9 +72,11 @@
   [p] {:name (util/search-normalize p (state/enable-search-remove-accents?))
        :original-name p})
 
-(defn make-pages-indice!
-  "Build a page indice from scratch.
-   Incremental page indice is implemented in frontend.search.sync-search-indice!"
+(defn make-pages-title-indice!
+  "Build a page title indice from scratch.
+   Incremental page title indice is implemented in frontend.search.sync-search-indice!
+   Rename from the page indice since 10.25.2022, since this is only used for page title search.
+   From now on, page indice is talking about page content search."
   []
   (when-let [repo (state/get-current-repo)]
     (let [pages (->> (db/get-pages (state/get-current-repo))

src/main/frontend/search/node.cljs

@@ -17,12 +17,22 @@
                 {:block/uuid uuid
                  :block/content content
                  :block/page page})) result)))
+  (query-page [_this q opts]
+    (p/let [result (ipc/ipc "search-pages" repo q opts)
+            result (bean/->clj result)]
+      (keep (fn [{:keys [content snippet uuid]}]
+              (when-not (> (count content) (* 10 (state/block-content-max-length repo)))
+                {:block/uuid uuid
+                 :block/snippet snippet})) result)))
   (rebuild-blocks-indice! [_this]
-    (let [indice (search-db/build-blocks-indice repo)]
-      (ipc/ipc "rebuild-blocks-indice" repo indice)))
+    (let [blocks-indice (search-db/build-blocks-indice repo)
+          pages-indice  (search-db/build-pages-indice repo)]
+      (ipc/ipc "rebuild-indice" repo blocks-indice pages-indice)))
   (transact-blocks! [_this data]
     (ipc/ipc "transact-blocks" repo (bean/->js data)))
   (truncate-blocks! [_this]
-    (ipc/ipc "truncate-blocks" repo))
+    (ipc/ipc "truncate-indice" repo))
+  (transact-pages! [_this data]
+    (ipc/ipc "transact-pages" repo (bean/->js data)))
   (remove-db! [_this]
     (ipc/ipc "remove-db" repo)))

src/main/frontend/search/plugin.cljs

@@ -23,6 +23,9 @@
   (query [_this q opts]
     (call-service! service "search:query" (merge {:q q} opts) true))
 
+  (query-page [_this q opts]
+    (call-service! service "search:queryPage" (merge {:q q} opts) true))
+
   (rebuild-blocks-indice! [_this]
    ;; Not pushing all data for performance temporarily
    ;;(let [blocks (search-db/build-blocks-indice repo)])
@@ -34,6 +37,12 @@
                      {:data {:added   blocks-to-add
                              :removed blocks-to-remove-set}})))
 
+  (transact-pages! [_this data]
+    (let [{:keys [pages-to-remove-set pages-to-add]} data]
+      (call-service! service "search:transactpages"
+                     {:data {:added   pages-to-add
+                             :removed pages-to-remove-set}})))
+
   (truncate-blocks! [_this]
     (call-service! service "search:truncateBlocks" {}))
 

src/main/frontend/search/protocol.cljs

@@ -1,8 +1,10 @@
 (ns ^:no-doc frontend.search.protocol)
 
 (defprotocol Engine
-  (query [this q option])
-  (rebuild-blocks-indice! [this])
+  (query [this q option]) 
+  (query-page [this q option])
+  (rebuild-blocks-indice! [this]) ;; TODO: rename to rebuild-indice!
   (transact-blocks! [this data])
-  (truncate-blocks! [this])
+  (truncate-blocks! [this]) ;; TODO: rename to truncate-indice!
+  (transact-pages! [this data])
   (remove-db! [this]))

src/main/frontend/state.cljs

@@ -51,7 +51,7 @@
      :journals-length                       3
 
      :search/q                              ""
-     :search/mode                           :global
+     :search/mode                           :global  ;; inner page or full graph? {:page :global}
      :search/result                         nil
      :search/graph-filters                  []
      :search/engines                        {}

src/main/frontend/util/text.cljs

@@ -118,6 +118,26 @@
              []
              ks))))
 
+(defn cut-by
+  "Cut string by specifid wrapping symbols, only match the first occurrence.
+     value - string to cut
+     before - cutting symbol (before)
+     end - cutting symbol (end)"
+  [value before end]
+  (let [b-pos (string/index-of value before)
+        b-len (count before)]
+    (if b-pos
+      (let [b-cut (subs value 0 b-pos)
+            m-cut (subs value (+ b-pos b-len))
+            e-len (count end)
+            e-pos (string/index-of m-cut end)]
+        (if e-pos
+          (let [e-cut (subs m-cut (+ e-pos e-len))
+                m-cut (subs m-cut 0 e-pos)]
+            [b-cut m-cut e-cut])
+          [b-cut m-cut nil]))
+      [value nil nil])))
+
 (defn get-graph-name-from-path
   [path]
   (when (string? path)

src/test/frontend/util/text_test.cljs

@@ -57,3 +57,37 @@
     '(false false false false false false true true true true true true)
     (map #(text-util/wrapped-by? "prop::value" % "::" "") (take 12 (range)))
     ))
+
+
+(deftest test-cut-by
+  []
+  (are [x y] (= x y)
+    ["" "" ""]
+    (text-util/cut-by "[[]]" "[[" "]]")
+
+    ["" "abc" ""]
+    (text-util/cut-by "[[abc]]" "[[" "]]")
+
+    ["012 " "6" " [[2]]"]
+    (text-util/cut-by "012 [[6]] [[2]]" "[[" "]]")
+
+    ["" "prop" "value"]
+    (text-util/cut-by "prop::value" "" "::")
+
+    ["prop" "" "value"]
+    (text-util/cut-by "prop::value" "::" "")
+
+    ["some " "content" " here"]
+    (text-util/cut-by "some $pfts>$content$pfts<$ here" "$pfts>$" "$pfts<$")
+
+    ["some " "content$pft" nil]
+    (text-util/cut-by "some $pfts>$content$pft" "$pfts>$" "$pfts<$")
+
+    ["some $pf" nil nil]
+    (text-util/cut-by "some $pf" "$pfts>$" "$pfts<$")
+
+    ["" "content" ""]
+    (text-util/cut-by "$pfts>$content$pfts<$" "$pfts>$" "$pfts<$")
+    
+    ["" "content$p" nil]
+    (text-util/cut-by "$pfts>$content$p" "$pfts>$" "$pfts<$")))