Perf/db serialization (#1417)

* perf: disable auto-save dbs on electron

We still save the dbs when user reload the app or close the app.

Notice: Ctrl-c doesn't work now

* chore: re-enable mtime check when writing to disk

* ui: add db persistent notification

resources/css/common.css

@@ -724,6 +724,7 @@ a.fade-link:hover {
 #head .refresh svg {
     height: 20px;
 }
+
 /* < > buttons */
 
 a.navigation {

src/electron/electron/core.cljs

@@ -5,7 +5,9 @@
             [clojure.string :as string]
             ["fs" :as fs]
             ["path" :as path]
-            ["electron" :refer [BrowserWindow app protocol ipcMain] :as electron]))
+            ["electron" :refer [BrowserWindow app protocol ipcMain] :as electron]
+            [clojure.core.async :as async]
+            [electron.state :as state]))
 
 (def ROOT_PATH (path/join js/__dirname ".."))
 (def MAIN_WINDOW_ENTRY (str "file://" (path/join js/__dirname (if dev? "electron-dev.html" "electron.html"))))
@@ -91,13 +93,20 @@
     #(do (.removeHandler ipcMain toggle-win-channel)
          (.removeHandler ipcMain call-app-channel))))
 
+(defonce *win (atom nil))
+
+(defn- destroy-window!
+  [^js win]
+  (.destroy win))
+
 (defn main
   []
-  (.on app "window-all-closed" #(when-not mac? (.quit app)))
+  (.on app "window-all-closed" (fn []
+                                 (when-not mac? (.quit app))))
   (.on app "ready"
        (fn []
          (let [^js win (create-main-window)
-               *win (atom win)
+               _ (reset! *win win)
                *quitting? (atom false)]
 
            (.. logger (info (str "Logseq App(" (.getVersion app) ") Starting... ")))
@@ -117,11 +126,20 @@
            (@*setup-fn)
 
            ;; main window events
-           (.on win "close" #(if (or @*quitting? (not mac?))
-                               (reset! *win nil)
-                               (do (.preventDefault ^js/Event %)
-                                   (.hide win))))
-           (.on app "before-quit" #(reset! *quitting? true))
+           (.on win "close" (fn [e]
+                              (.preventDefault e)
+                              (let [web-contents (. win -webContents)]
+                                (.send web-contents "persistent-dbs"))
+                              (async/go
+                                ;; FIXME: What if persistence failed?
+                                (let [_ (async/<! state/persistent-dbs-chan)]
+                                  (if (or @*quitting? (not mac?))
+                                    (when-let [win @*win]
+                                      (destroy-window! win)
+                                      (reset! *win nil))
+                                    (do (.preventDefault ^js/Event e)
+                                        (.hide win)))))))
+           (.on app "before-quit" (fn [_e] (reset! *quitting? true)))
            (.on app "activate" #(if @*win (.show win)))))))
 
 (defn start []

src/electron/electron/handler.cljs

@@ -7,7 +7,9 @@
             [promesa.core :as p]
             [goog.object :as gobj]
             [clojure.string :as string]
-            [electron.utils :as utils]))
+            [electron.utils :as utils]
+            [electron.state :as state]
+            [clojure.core.async :as async]))
 
 (defmulti handle (fn [_window args] (keyword (first args))))
 
@@ -79,6 +81,10 @@
 (defmethod handle :getFiles [window [_ path]]
   (get-files path))
 
+(defmethod handle :persistent-dbs-saved [window _]
+  (async/put! state/persistent-dbs-chan true )
+  true)
+
 (defn- get-file-ext
   [file]
   (last (string/split file #"\.")))

src/electron/electron/state.cljs

@@ -0,0 +1,4 @@
+(ns electron.state
+  (:require [clojure.core.async :as async]))
+
+(defonce persistent-dbs-chan (async/chan 1))

src/main/electron/listener.cljs

@@ -2,7 +2,13 @@
   (:require [frontend.state :as state]
             [frontend.handler.route :as route-handler]
             [cljs-bean.core :as bean]
-            [frontend.fs.watcher-handler :as watcher-handler]))
+            [frontend.fs.watcher-handler :as watcher-handler]
+            [frontend.db :as db]
+            [frontend.idb :as idb]
+            [promesa.core :as p]
+            [electron.ipc :as ipc]
+            [frontend.handler.notification :as notification]
+            [frontend.ui :as ui]))
 
 (defn listen-to-open-dir!
   []
@@ -20,7 +26,33 @@
                        (let [{:keys [type payload]} (bean/->clj data)]
                          (watcher-handler/handle-changed! type payload)))))
 
+(defn listen-persistent-dbs!
+  []
+  ;; TODO: move "file-watcher" to electron.ipc.channels
+  (js/window.apis.on
+   "persistent-dbs"
+   (fn [req]
+     (p/let [repos (idb/get-nfs-dbs)
+             repos (-> repos
+                       (conj (state/get-current-repo))
+                       (distinct))]
+       (if (seq repos)
+         (do
+           (notification/show!
+            (ui/loading "Logseq is saving the graphs to your local file system, please wait for several seconds.")
+            :warning)
+           (js/setTimeout
+            (fn []
+              (-> (p/all (map db/persist! repos))
+                 (p/then (fn []
+                           (ipc/ipc "persistent-dbs-saved")))
+                 (p/catch (fn [error]
+                            (js/console.dir error)))))
+            100))
+         (ipc/ipc "persistent-dbs-saved"))))))
+
 (defn listen!
   []
   (listen-to-open-dir!)
-  (run-dirs-watcher!))
+  (run-dirs-watcher!)
+  (listen-persistent-dbs!))

src/main/frontend/components/header.cljs

@@ -87,7 +87,9 @@
           :options {:href (rfe/href :graph)}
           :icon svg/graph-sm})
 
-       (when (or logged? (and (nfs/supported?) current-repo))
+       (when (or logged?
+                 (util/electron?)
+                 (and (nfs/supported?) current-repo))
          {:title (t :all-graphs)
           :options {:href (rfe/href :repos)}
           :icon svg/repos-sm})

src/main/frontend/db.cljs

@@ -108,10 +108,11 @@
   [repo conn files-db?]
   (d/listen! conn :persistence
              (fn [tx-report]
-               (let [tx-id (get-tx-id tx-report)]
-                 (state/set-last-transact-time! repo (util/time-ms))
-                 ;; (state/persist-transaction! repo files-db? tx-id (:tx-data tx-report))
-                 (persist-if-idle! repo))
+               (when-not (util/electron?)
+                (let [tx-id (get-tx-id tx-report)]
+                  (state/set-last-transact-time! repo (util/time-ms))
+                  ;; (state/persist-transaction! repo files-db? tx-id (:tx-data tx-report))
+                  (persist-if-idle! repo)))
 
                ;; rebuild search indices
                (when-not files-db?

src/main/frontend/db/conn.cljs

@@ -88,7 +88,8 @@
 
      (d/transact! db-conn default-db/built-in-pages)
 
-     (when listen-handler (listen-handler repo)))))
+     (when (and listen-handler (not (util/electron?)))
+       (listen-handler repo)))))
 
 (defn destroy-all!
   []

src/main/frontend/fs/node.cljs

@@ -25,34 +25,22 @@
 
 (defn- write-file-impl!
   [repo dir path content {:keys [ok-handler error-handler] :as opts} stat]
-  (->
-   (p/let [result (ipc/ipc "writeFile" path content)
-           mtime (gobj/get result "mtime")]
-     (db/set-file-last-modified-at! repo path mtime)
-     (when ok-handler
-       (ok-handler repo path result))
-     result)
-   (p/catch (fn [error]
-              (if error-handler
-                (error-handler error)
-                (log/error :write-file-failed error)))))
-  ;; (p/let [disk-mtime (when stat (gobj/get stat "mtime"))
-  ;;         db-mtime (db/get-file-last-modified-at repo path)]
-  ;;   (if (not= disk-mtime db-mtime)
-  ;;     (js/alert (str "The file has been modified in your local disk! File path: " path
-  ;;                    ", please save your changes and click the refresh button to reload it."))
-  ;;     (->
-  ;;      (p/let [result (ipc/ipc "writeFile" path content)
-  ;;              mtime (gobj/get result "mtime")]
-  ;;        (db/set-file-last-modified-at! repo path mtime)
-  ;;        (when ok-handler
-  ;;          (ok-handler repo path result))
-  ;;        result)
-  ;;      (p/catch (fn [error]
-  ;;                 (if error-handler
-  ;;                   (error-handler error)
-  ;;                   (log/error :write-file-failed error)))))))
-)
+  (p/let [disk-mtime (when stat (gobj/get stat "mtime"))
+          db-mtime (db/get-file-last-modified-at repo path)]
+    (if (not= disk-mtime db-mtime)
+      (js/alert (str "The file has been modified in your local disk! File path: " path
+                     ", please save your changes and click the refresh button to reload it."))
+      (->
+       (p/let [result (ipc/ipc "writeFile" path content)
+               mtime (gobj/get result "mtime")]
+         (db/set-file-last-modified-at! repo path mtime)
+         (when ok-handler
+           (ok-handler repo path result))
+         result)
+       (p/catch (fn [error]
+                  (if error-handler
+                    (error-handler error)
+                    (log/error :write-file-failed error))))))))
 
 (defrecord Node []
   protocol/Fs

src/main/frontend/fs/watcher_handler.cljs

@@ -40,20 +40,9 @@
                (> mtime last-modified-at)))
 
         (let [_ (file-handler/alter-file repo path content {:re-render-root? true
-                                                              :from-disk? true})]
+                                                            :from-disk? true})]
           (db/set-file-last-modified-at! repo path mtime))
 
-        ;; (= "unlink" type)
-        ;; (when-let [page-name (db/get-file-page path)]
-        ;;   (page-handler/delete!
-        ;;    page-name
-        ;;    (fn []
-        ;;      (notification/show! (str "Page " page-name " was deleted on disk.")
-        ;;                          :success)
-        ;;      (when (= (state/get-current-page) page-name)
-        ;;        ;; redirect to home
-        ;;        (route-handler/redirect-to-home!)))))
-
         (contains? #{"add" "change" "unlink"} type)
         nil
 

src/main/frontend/handler.cljs

@@ -124,6 +124,39 @@
   (js/window.addEventListener "online" handle-connection-change)
   (js/window.addEventListener "offline" handle-connection-change))
 
+(defn- get-repos
+  []
+  (let [logged? (state/logged?)
+        me (state/get-me)]
+    (p/let [nfs-dbs (idb/get-nfs-dbs)
+            nfs-dbs (map (fn [db]
+                           {:url db :nfs? true}) nfs-dbs)]
+      (cond
+        logged?
+        (concat
+         nfs-dbs
+         (:repos me))
+
+        (seq nfs-dbs)
+        nfs-dbs
+
+        :else
+        [{:url config/local-repo
+          :example? true}]))))
+
+(defn persist-dbs-when-reload!
+  []
+  (p/let [repos (get-repos)]
+    (let [repos (-> (map :url repos)
+                    (conj (state/get-current-repo))
+                    (distinct))
+          repos (remove nil? repos)]
+      (-> (p/all (map db/persist! repos))
+          (p/then (fn []
+                    (println "DB persisted.")))
+          (p/catch (fn [error]
+                     (js/console.dir error)))))))
+
 (defn start!
   [render]
   (let [{:keys [me logged? repos]} (get-me-and-repos)]
@@ -138,26 +171,17 @@
        (notification/show! "Sorry, it seems that your browser doesn't support IndexedDB, we recommend to use latest Chrome(Chromium) or Firefox(Non-private mode)." :error false)
        (state/set-indexedb-support! false)))
 
-    (p/let [nfs-dbs (idb/get-nfs-dbs)
-            nfs-dbs (map (fn [db]
-                           {:url db :nfs? true}) nfs-dbs)]
-      (let [repos (cond
-                    logged?
-                    (concat
-                     nfs-dbs
-                     (:repos me))
-
-                    (seq nfs-dbs)
-                    nfs-dbs
-
-                    :else
-                    [{:url config/local-repo
-                      :example? true}])]
-        (state/set-repos! repos)
-        (restore-and-setup! me repos logged?)))
+    (p/let [repos (get-repos)]
+      (state/set-repos! repos)
+      (restore-and-setup! me repos logged?))
     (reset! db/*sync-search-indice-f search/sync-search-indice!)
     (db/run-batch-txs!)
     (file-handler/run-writes-chan!)
     (editor-handler/periodically-save!)
     (when (util/electron?)
-      (el/listen!))))
+      (el/listen!))
+    ;; Disable reload
+    (set! (.-onbeforeunload js/window)
+          (fn [^js e]
+            (persist-dbs-when-reload!)
+            (set! (.-returnValue e) false)))))

src/main/frontend/ui.cljs

@@ -138,7 +138,7 @@
                 :stroke-linejoin "round"
                 :stroke-linecap  "round"}]]]
             :warning
-            ["text-gray-900"
+            ["text-gray-900 dark:text-gray-300 "
              [:svg.h-6.w-6.text-yellow-500
               {:stroke "currentColor", :viewBox "0 0 24 24", :fill "none"}
               [:path