enhance: add :graph/ready event; check search version of each repo

src/electron/electron/core.cljs

@@ -201,9 +201,6 @@
 
                (utils/disableXFrameOptions win)
 
-               (when (search/version-changed?)
-                 (search/rm-search-dir!))
-
                (search/ensure-search-dir!)
 
                (search/open-dbs!)

src/electron/electron/handler.cljs

@@ -232,6 +232,7 @@
   (search/truncate-blocks-table! repo)
   ;; unneeded serialization
   (search/upsert-blocks! repo (bean/->js data))
+  (search/write-search-version! repo)
   [])
 
 (defmethod handle :transact-blocks [_window [_ repo data]]
@@ -352,6 +353,10 @@
         windows (filter #(.isVisible %) windows)]
     (> (count windows) 1)))
 
+(defmethod handle :searchVersionChanged?
+  [^js _win [_ graph]]
+  (search/version-changed? graph))
+
 (defmethod handle :reloadWindowPage [^js win]
   (when-let [web-content (.-webContents win)]
     (.reload web-content)))

src/electron/electron/search.cljs

@@ -5,7 +5,11 @@
             [clojure.string :as string]
             ["electron" :refer [app]]))
 
-(defonce version "0.0.1")
+;; version of the search cache
+;; ver. 0.0.1: initial version
+;; ver. 0.0.2: bump version as page name breaking changes of LogSeq 0.5.7 ~ 0.5.9
+(defonce version "0.0.2")
+(defonce invalid-version "0.0.0")
 
 (defonce databases (atom nil))
 
@@ -52,8 +56,8 @@
     END;"
                   ]]
     (doseq [trigger triggers]
-     (let [stmt (prepare db trigger)]
-       (.run ^object stmt)))))
+      (let [stmt (prepare db trigger)]
+        (.run ^object stmt)))))
 
 (defn create-blocks-table!
   [db]
@@ -76,33 +80,16 @@
 
 (defonce search-version "search.version")
 
-(defn get-search-version
-  []
-  (let [path (.getPath ^object app "userData")
-        path (path/join path search-version)]
-    (when (fs/existsSync path)
-      (.toString (fs/readFileSync path)))))
-
-(defn write-search-version!
-  []
-  (let [path (.getPath ^object app "userData")
-        path (path/join path search-version)]
-    (fs/writeFileSync path version)))
-
-(defn version-changed?
-  []
-  (not= version (get-search-version)))
 
 (defn ensure-search-dir!
   []
-  (write-search-version!)
   (fs/ensureDirSync (get-search-dir)))
 
-(defn rm-search-dir!
-  []
-  (let [search-dir (get-search-dir)]
-    (when (fs/existsSync search-dir)
-      (fs/removeSync search-dir))))
+;; (defn rm-search-dir!
+;;   []
+;;   (let [search-dir (get-search-dir)]
+;;     (when (fs/existsSync search-dir)
+;;       (fs/removeSync search-dir))))
 
 (defn get-db-full-path
   [db-name]
@@ -110,19 +97,44 @@
         search-dir (get-search-dir)]
     [db-name (path/join search-dir db-name)]))
 
+(defn get-db-version-path
+  "File for storing search cache version"
+  [db-name]
+  (let [[db-name db-full-path] (get-db-full-path db-name)]
+    [db-name db-full-path (str db-full-path "." search-version)]))
+
+(defn get-search-version
+  "Not normalized db name"
+  [db-name]
+  (let [[_db-name db-full-path db-ver-path] (get-db-version-path db-name)]
+    (if (and (fs/existsSync db-ver-path) (fs/existsSync db-full-path)) ;; avoid case that only ver file exists
+      (.toString (fs/readFileSync db-ver-path))
+      invalid-version))) ;; no any cache exists
+
+(defn write-search-version!
+  [db-name]
+  (let [[_db-name _db-full-path db-ver-path] (get-db-version-path db-name)]
+    (fs/writeFileSync db-ver-path version)))
+
+(defn version-changed?
+  [db-name]
+  (not= version (get-search-version db-name)))
+
 (defn open-db!
   [db-name]
-  (let [[db-name db-full-path] (get-db-full-path db-name)
+  (let [[db-normalized-name db-full-path] (get-db-full-path db-name)
         db (sqlite3 db-full-path nil)
         _ (create-blocks-table! db)
         _ (create-blocks-fts-table! db)
         _ (add-triggers! db)]
-    (swap! databases assoc db-name db)))
+    (swap! databases assoc db-normalized-name db)))
 
 (defn open-dbs!
   []
   (let [search-dir (get-search-dir)
-        dbs (fs/readdirSync search-dir)]
+        dbs (fs/readdirSync search-dir)
+        dbs (remove (fn [file-name] (.endsWith file-name search-version)) dbs)
+        dbs (remove (fn [file-name] (.startsWith file-name ".")) dbs)]
     (when (seq dbs)
       (doseq [db-name dbs]
         (open-db! db-name)))))
@@ -145,7 +157,7 @@
   [repo ids]
   (when-let [db (get-db repo)]
     (let [ids (->> (map (fn [id] (str "'" id "'")) ids)
-               (string/join ", "))
+                   (string/join ", "))
           sql (str "DELETE from blocks WHERE id IN (" ids ")")
           stmt (prepare db sql)]
       (.run ^object stmt))))
@@ -211,9 +223,10 @@
   [repo]
   (when-let [database (get-db repo)]
     (.close database)
-    (let [[db-name db-full-path] (get-db-full-path repo)]
+    (let [[db-name db-full-path db-ver-path] (get-db-version-path repo)]
       (println "Delete search indice: " db-full-path)
       (fs/unlinkSync db-full-path)
+      (fs/unlinkSync db-ver-path)
       (swap! databases dissoc db-name))))
 
 (defn query
@@ -225,7 +238,6 @@
 (comment
   (def repo (first (keys @databases)))
   (query repo
-    "select * from blocks_fts")
+         "select * from blocks_fts")
 
-  (delete-db! repo)
-  )
+  (delete-db! repo))

src/main/frontend/db.cljs

@@ -157,7 +157,7 @@
                        :listen-handler listen-and-persist!))))
 
 ;; TODO: only restore the current graph instead of all the graphs to speedup and
-;; reduce memory usage.
+;; reduce memory usage. pub event :graph/ready when a graph is restored, and finish the TODOs in :graph/ready
 (defn restore!
   [{:keys [repos] :as me} _old-db-schema restore-config-handler]
   (let [logged? (:name me)]

src/main/frontend/handler.cljs

@@ -133,6 +133,7 @@
                                        (mobile-util/native-ios?))
                               (mobile-util/icloud-sync!))
                             (file-handler/watch-for-current-graph-dir!)))
+                         (p/then (state/pub-event! [:graph/ready (state/get-current-repo)]))
                          (p/catch (fn [error]
                                     (log/error :exception error))))))
         interval-id (js/setInterval inner-fn 50)]

src/main/frontend/handler/events.cljs

@@ -2,7 +2,7 @@
   (:refer-clojure :exclude [run!])
   (:require [clojure.core.async :as async]
             [clojure.set :as set]
-            [datascript.core :as d]
+            ;; [datascript.core :as d]
             [frontend.components.diff :as diff]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.components.plugins :as plugin]
@@ -13,7 +13,7 @@
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.db-schema :as db-schema]
-            [frontend.db.conn :as conn]
+            ;; [frontend.db.conn :as conn]
             [frontend.extensions.srs :as srs]
             [frontend.fs.nfs :as nfs]
             [frontend.handler.common :as common-handler]
@@ -32,7 +32,7 @@
             [frontend.ui :as ui]
             [frontend.util :as util]
             [rum.core :as rum]
-            ["semver" :as semver]
+            ;; ["semver" :as semver]
             [frontend.modules.instrumentation.posthog :as posthog]
             [frontend.mobile.util :as mobile-util]
             [frontend.encrypt :as encrypt]
@@ -89,7 +89,8 @@
     (route-handler/redirect-to-home!))
   (when-let [dir-name (config/get-repo-dir graph)]
     (fs/watch-dir! dir-name))
-  (srs/update-cards-due-count!))
+  (srs/update-cards-due-count!)
+  (state/pub-event! [:graph/ready graph]))
 
 (defmethod handle :graph/switch [[_ graph]]
   (if (outliner-file/writes-finished?)
@@ -207,20 +208,27 @@
   (p/let [content (when content (encrypt/decrypt content))]
     (state/set-modal! #(git-component/file-specific-version path hash content))))
 
-(defmethod handle :after-db-restore [[_ repos]]
-  (mapv (fn [{url :url}]
-          ;; compare :ast/version
-          (let [db (conn/get-conn url)
-                ast-version (:v (first (d/datoms db :aevt :ast/version)))]
-            (when (and (not= config/local-repo url)
-                       (or (nil? ast-version)
-                           (. semver lt ast-version db-schema/ast-version)))
-              (notification/show!
-               [:p.content
-                (util/format "DB-schema updated, Please re-index repo [%s]" url)]
-               :warning
-               false))))
-        repos))
+;; defunced in https://github.com/logseq/logseq/commit/097d47a860dd234715d670a78f61dddfe7ca698f
+;; (defmethod handle :after-db-restore [[_ repos]]
+;;   (mapv (fn [{url :url}]
+;;           ;; compare :ast/version
+;;           (let [db (conn/get-conn url)
+;;                 ast-version (:v (first (d/datoms db :aevt :ast/version)))]
+;;             (when (and (not= config/local-repo url)
+;;                        (or (nil? ast-version)
+;;                            (. semver lt ast-version db-schema/ast-version)))
+;;               (notification/show!
+;;                [:p.content
+;;                 (util/format "DB-schema updated, Please re-index repo [%s]" url)]
+;;                :warning
+;;                false))))
+;;         repos))
+
+;; TODO: when "only restore the current graph instead of all the graphs" is done,
+;; remove invoke of :graph/ready in graph/switch and restore-and-setup!
+(defmethod handle :graph/ready [[_ repo]]
+  (js/console.log "graph ready")
+  (search-handler/rebuild-indices-when-stale! repo))
 
 (defmethod handle :notification/show [[_ {:keys [content status clear?]}]]
   (notification/show! content status clear?))

src/main/frontend/handler/search.cljs

@@ -70,3 +70,15 @@
        (notification-handler/show!
         "Search indices rebuilt successfully!"
         :success)))))
+
+(defn rebuild-indices-when-stale!
+  ([]
+   (rebuild-indices-when-stale! (state/get-current-repo)))
+  ([repo]
+   (p/let [cache-stale? (search/cache-stale? repo)]
+     (when cache-stale?
+       (js/console.log "cache stale: " repo)
+       (p/let [_ (search/rebuild-indices! repo)]
+         (notification-handler/show!
+          "Stale search cache detected. Search indices rebuilt successfully!"
+          :success))))))

src/main/frontend/search.cljs

@@ -233,3 +233,8 @@
   [repo]
   (when-let [engine (get-engine repo)]
     (protocol/remove-db! engine)))
+
+(defn cache-stale?
+  [repo]
+  (when-let [engine (get-engine repo)]
+    (protocol/cache-stale? engine repo)))

src/main/frontend/search/browser.cljs

@@ -37,6 +37,7 @@
   (rebuild-blocks-indice! [_this]
     (let [indice (search-db/make-blocks-indice! repo)]
       (p/promise indice)))
+  (cache-stale? [_this _repo] (p/promise false)) ;; fuse.js doesn't have cache
   (transact-blocks! [_this {:keys [blocks-to-remove-set
                                   blocks-to-add]}]
     (swap! search-db/indices update-in [repo :blocks]

src/main/frontend/search/db.cljs

@@ -6,6 +6,8 @@
             [frontend.util :as util]
             ["fuse.js" :as fuse]))
 
+;; Notice: When breaking changes happen, bump version in src/electron/electron/search.cljs
+
 (defonce indices (atom nil))
 
 (defn block->index

src/main/frontend/search/node.cljs

@@ -14,6 +14,9 @@
              {:block/uuid uuid
               :block/content content
               :block/page page}) result)))
+  (cache-stale? [_this repo]
+                ;; only FTS require cache validating
+                (ipc/ipc "searchVersionChanged?" repo))
   (rebuild-blocks-indice! [_this]
     (let [indice (search-db/build-blocks-indice repo)]
       (ipc/ipc "rebuild-blocks-indice" repo indice)))

src/main/frontend/search/protocol.cljs

@@ -2,6 +2,7 @@
 
 (defprotocol Engine
   (query [this q option])
+  (cache-stale? [this repo])
   (rebuild-blocks-indice! [this])
   (transact-blocks! [this data])
   (truncate-blocks! [this])