add page indice logics

src/electron/electron/handler.cljs

@@ -285,38 +285,42 @@
   (async/put! state/persistent-dbs-chan true)
   true)
 
+;; Search related IPCs
 (defmethod handle :search-blocks [_window [_ repo q opts]]
   (search/search-blocks repo q opts))
 
-(defmethod handle :rebuild-blocks-indice [_window [_ repo data]]
+(defmethod handle :rebuild-indice [_window [_ repo block-data page-data]]
   (search/truncate-blocks-table! repo)
   ;; unneeded serialization
-  (search/upsert-blocks! repo (bean/->js data))
+  (search/upsert-blocks! repo (bean/->js block-data))
+  (search/truncate-pages-table! repo)
+  (search/upsert-pages! repo (bean/->js page-data))
   [])
 
 (defmethod handle :transact-blocks [_window [_ repo data]]
-  (let [{:keys [blocks-to-remove-set blocks-to-add]} data
-        affected-pages-by-removal (when (seq blocks-to-remove-set)
-                                    ;; Be careful about Side Effect! Execution order matters!
-                                    ;; Should query before blocks are deleted in 
-                                    (search/get-page-by-block-ids repo blocks-to-remove-set))
-        affected-pages (->> (map :page blocks-to-add)
-                            (concat affected-pages-by-removal)
-                            distinct)]
+  (let [{:keys [blocks-to-remove-set blocks-to-add]} data]
     (when (seq blocks-to-remove-set)
       (search/delete-blocks! repo blocks-to-remove-set))
     (when (seq blocks-to-add)
       ;; unneeded serialization
-      (search/upsert-blocks! repo (bean/->js blocks-to-add)))
-    (when (seq affected-pages)
-      ;; TODO Junyi: update pages
-      (prn "affected-pages: " affected-pages))))
+      (search/upsert-blocks! repo (bean/->js blocks-to-add)))))
 
-(defmethod handle :truncate-blocks [_window [_ repo]]
-  (search/truncate-blocks-table! repo))
+(defmethod handle :transact-pages [_window [_ repo data]]
+  (let [{:keys [pages-to-remove-set pages-to-add]} data]
+    (when (seq pages-to-remove-set)
+      (search/delete-pages! repo pages-to-remove-set))
+    (when (seq pages-to-add)
+      ;; unneeded serialization
+      (search/upsert-pages! repo (bean/->js pages-to-add)))))
+
+(defmethod handle :truncate-indice [_window [_ repo]]
+  (search/truncate-blocks-table! repo)
+  (search/truncate-pages-table! repo))
 
 (defmethod handle :remove-db [_window [_ repo]]
   (search/delete-db! repo))
+;; ^^^^
+;; Search related IPCs End
 
 (defn clear-cache!
   [window]

src/electron/electron/search.cljs

@@ -5,8 +5,7 @@
             ["better-sqlite3" :as sqlite3]
             [clojure.string :as string]
             ["electron" :refer [app]]
-            [electron.logger :as logger]
-            [cljs-bean.core :as bean]))
+            [electron.logger :as logger]))
 
 (defonce databases (atom nil))
 
@@ -165,24 +164,26 @@
   (str "(" (->> (map (fn [id] (str "'" id "'")) ids)
                 (string/join ", ")) ")"))
 
-(defn get-page-by-block-ids
-  "Return a clojure list of the page ids that contains the provided block ids (distinct)"
-  [repo block-ids]
+(defn upsert-pages!
+  [repo pages]
   (if-let [db (get-db repo)]
     ;; TODO: what if a CONFLICT on uuid
-    (let [sql   (str "SELECT DISTINCT page from blocks WHERE id IN " (clj-list->sql block-ids))
-          _ (prn sql) ;; TODO Junyi
-          query (prepare db sql)]
-      (->> (.all ^object query)
-           bean/->clj
-           (map :page)))
+    (let [insert (prepare db "INSERT INTO pages (id, uuid, content) VALUES (@id, @uuid, @content) ON CONFLICT (id) DO UPDATE SET content = @content")
+          insert-many (.transaction ^object db
+                                    (fn [pages]
+                                      (doseq [page pages]
+                                        (.run ^object insert page))))]
+      (insert-many pages))
     (do
       (open-db! repo)
-      (get-page-by-block-ids repo block-ids))))
+      (upsert-pages! repo pages))))
 
-(defn upsert-pages!
-  [repo page-ids]
-  (prn page-ids))
+(defn delete-pages!
+  [repo ids]
+  (when-let [db (get-db repo)]
+    (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))
+          stmt (prepare db sql)]
+      (.run ^object stmt))))
 
 (defn upsert-blocks!
   [repo blocks]
@@ -203,9 +204,7 @@
   (when-let [db (get-db repo)]
     (let [sql (str "DELETE from blocks WHERE id IN " (clj-list->sql ids))
           stmt (prepare db sql)]
-      (.run ^object stmt))
-    ;; TODO Junyi: when blocks are updated, update the page index on demand
-    (upsert-pages! repo ids)))
+      (.run ^object stmt))))
 
 ;; (defn search-blocks-fts
 ;;   [q]
@@ -295,6 +294,16 @@
                         "delete from blocks_fts;")]
       (.run ^object stmt))))
 
+(defn truncate-pages-table!
+  [repo]
+  (when-let [database (get-db repo)]
+    (let [stmt (prepare database
+                        "delete from pages;")
+          _ (.run ^object stmt)
+          stmt (prepare database
+                        "delete from pages_fts;")]
+      (.run ^object stmt))))
+
 (defn delete-db!
   [repo]
   (when-let [database (get-db repo)]
@@ -309,9 +318,3 @@
   (when-let [database (get-db repo)]
     (let [stmt (prepare database sql)]
       (.all ^object stmt))))
-
-(comment
-  (def repo (first (keys @databases)))
-  (query repo
-         "select * from blocks_fts")
-  (delete-db! repo))

src/main/frontend/db/model.cljs

@@ -1528,6 +1528,7 @@
             assets (get-assets datoms)]
         [@(d/conn-from-datoms datoms db-schema/schema) assets]))))
 
+;; Deprecated?
 (defn delete-blocks
   [repo-url files _delete-page?]
   (when (seq files)

src/main/frontend/search.cljs

@@ -10,6 +10,7 @@
             [frontend.search.agency :as search-agency]
             [frontend.search.db :as search-db :refer [indices]]
             [frontend.search.protocol :as protocol]
+            [frontend.modules.datascript-report.core :as ds-report]
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.util.property :as property]
@@ -100,6 +101,12 @@
   (when-let [engine (get-engine repo)]
     (protocol/transact-blocks! engine data)))
 
+;; TODO Junyi: Plug-in the call
+(defn- transact-pages!
+  [repo data]
+  (when-let [engine (get-engine repo)]
+    (protocol/transact-pages! engine data)))
+
 (defn exact-matched?
   "Check if two strings points toward same search result"
   [q match]
@@ -124,7 +131,7 @@
            q (clean-str q)]
        (when-not (string/blank? q)
          (let [indice (or (get-in @indices [repo :pages])
-                          (search-db/make-pages-indice!))
+                          (search-db/make-pages-title-indice!))
                result (->> (.search indice q (clj->js {:limit limit}))
                            (bean/->clj))]
            ;; TODO: add indexes for highlights
@@ -191,13 +198,15 @@
            (let [result (fuzzy-search result q :limit limit)]
              (vec result))))))))
 
+;; TODO merge with logic in `invoke-hooks` when feature and test is sufficient
 (defn sync-search-indice!
   [repo tx-report]
   (let [data (:tx-data tx-report)
         datoms (filter
                 (fn [datom]
                   (contains? #{:block/name :block/content} (:a datom)))
-                data)]
+                data)
+        diff-pages (:pages (ds-report/get-blocks-and-pages tx-report))]
     (when (seq datoms)
       (let [datoms (group-by :a datoms)
             pages (:block/name datoms)
@@ -251,10 +260,10 @@
   ([repo]
    (when repo
      (when-let [engine (get-engine repo)]
-       (let [pages (search-db/make-pages-indice!)]
+       (let [page-titles (search-db/make-pages-title-indice!)]
          (p/let [blocks (protocol/rebuild-blocks-indice! engine)]
-           (let [result {:pages pages
-                         :blocks blocks}]
+           (let [result {:pages         page-titles ;; TODO: rename key to :page-titles
+                         :blocks        blocks}]
              (swap! indices assoc repo result)
              indices)))))))
 

src/main/frontend/search/db.cljs

@@ -29,6 +29,16 @@
        (remove nil?)
        (bean/->js)))
 
+;; TODO Junyi: Finalize index code
+(defn build-pages-indice
+  ;; TODO: Remove repo effects fns further up the call stack. db fns need standardization on taking connection
+  #_:clj-kondo/ignore
+  [repo]
+  (->> (db/get-all-page-contents)
+       (map page->index)
+       (remove nil?)
+       (bean/->js)))
+
 (defn make-blocks-indice!
   [repo]
   (let [blocks (build-blocks-indice repo)
@@ -46,9 +56,11 @@
   [p] {:name (util/search-normalize p (state/enable-search-remove-accents?))
        :original-name p})
 
-(defn make-pages-indice!
-  "Build a page indice from scratch.
-   Incremental page indice is implemented in frontend.search.sync-search-indice!"
+(defn make-pages-title-indice!
+  "Build a page title indice from scratch.
+   Incremental page title indice is implemented in frontend.search.sync-search-indice!
+   Rename from the page indice since 10.25.2022, since this is only used for page title search.
+   From now on, page indice is talking about page content search."
   []
   (when-let [repo (state/get-current-repo)]
     (let [pages (->> (db/get-pages (state/get-current-repo))

src/main/frontend/search/node.cljs

@@ -18,11 +18,14 @@
                  :block/content content
                  :block/page page})) result)))
   (rebuild-blocks-indice! [_this]
-    (let [indice (search-db/build-blocks-indice repo)]
-      (ipc/ipc "rebuild-blocks-indice" repo indice)))
+    (let [blocks-indice (search-db/build-blocks-indice repo)
+          pages-indice  (search-db/build-pages-indice repo)]
+      (ipc/ipc "rebuild-indice" repo blocks-indice pages-indice)))
   (transact-blocks! [_this data]
     (ipc/ipc "transact-blocks" repo (bean/->js data)))
   (truncate-blocks! [_this]
-    (ipc/ipc "truncate-blocks" repo))
+    (ipc/ipc "truncate-indice" repo)) 
+  (transact-pages! [_this data]
+    (ipc/ipc "transact-pages" repo (bean/->js data)))
   (remove-db! [_this]
     (ipc/ipc "remove-db" repo)))

src/main/frontend/search/protocol.cljs

@@ -1,8 +1,9 @@
 (ns ^:no-doc frontend.search.protocol)
 
 (defprotocol Engine
-  (query [this q option])
-  (rebuild-blocks-indice! [this])
+  (query [this q option]) 
+  (rebuild-blocks-indice! [this]) ;; TODO: rename to rebuild-indice!
   (transact-blocks! [this data])
-  (truncate-blocks! [this])
+  (truncate-blocks! [this]) ;; TODO: rename to truncate-indice!
+  (transact-pages! [this data])
   (remove-db! [this]))