Refactor: use last modified time to decide if file has been changed

Previously, we're using file content.

Fixes LOG-3161

deps/db/src/logseq/db/frontend/schema.cljs

@@ -107,13 +107,13 @@
 
    ;; file
    :file/path {:db/unique :db.unique/identity}
+   :file/last-modified-at {}
    ;; only store the content of logseq's files
    :file/content {}
 
    ;; TODO: do we really use this?
    :file/handle {}
    ;; :file/created-at {}
-   ;; :file/last-modified-at {}
    ;; :file/size {}
    })
 
@@ -136,7 +136,6 @@
     :property/schema.classes {:db/valueType :db.type/ref
                               :db/cardinality :db.cardinality/many}
     :property.value/content {}
-    :file/last-modified-at {}
     :asset/uuid {:db/unique :db.unique/identity}
     :asset/meta {}}))
 

src/main/frontend/db.cljs

@@ -86,8 +86,7 @@
   (when (and repo path last-modified-at)
     (transact! repo
                [{:file/path path
-                 :file/last-modified-at last-modified-at}]
-               {:skip-refresh? true})))
+                 :file/last-modified-at last-modified-at}] {})))
 
 (defn set-file-content!
   ([repo path content]
@@ -96,4 +95,4 @@
    (when (and repo path)
      (let [tx-data {:file/path path
                     :file/content content}]
-       (transact! repo [tx-data] (merge opts {:skip-refresh? true}))))))
+       (transact! repo [tx-data] opts)))))

src/main/frontend/fs/watcher_handler.cljs

@@ -19,8 +19,7 @@
             [logseq.common.config :as common-config]
             [logseq.common.util.block-ref :as block-ref]
             [promesa.core :as p]
-            [frontend.db.async :as db-async]
-            [frontend.db.transact :as db-transact]))
+            [frontend.db.async :as db-async]))
 
 ;; all IPC paths must be normalized! (via common-util/path-normalize)
 
@@ -43,9 +42,12 @@
 (defn- handle-add-and-change!
   [repo path content db-content mtime backup?]
   (let [config (state/get-config repo)
-        path-hidden-patterns (:hidden config)]
-    (when-not (and (seq path-hidden-patterns)
-                   (common-config/hidden? path path-hidden-patterns))
+        path-hidden-patterns (:hidden config)
+        db-last-modified-at (db/get-file-last-modified-at repo path)]
+    (when-not (or (and (seq path-hidden-patterns)
+                    (common-config/hidden? path path-hidden-patterns))
+                  ;; File not changed
+                  (= db-last-modified-at mtime))
       (p/let [;; save the previous content in a versioned bak file to avoid data overwritten.
               _ (when backup?
                   (-> (when-let [repo-dir (config/get-local-dir repo)]
@@ -60,87 +62,77 @@
 
 (defn handle-changed!
   [type {:keys [dir path content stat global-dir] :as payload}]
-  (let [repo (state/get-current-repo)
-        ^js sqlite @state/*db-worker]
-    (p/let [writes-finished? (when sqlite (.file-writes-finished? sqlite repo))]
-      (when (and dir writes-finished? (db-transact/request-finished?))
-        (let [;; Global directory events don't know their originating repo so we rely
+  (let [repo (state/get-current-repo)]
+    (when dir
+      (let [;; Global directory events don't know their originating repo so we rely
           ;; on the client to correctly identify it
-              repo (cond
-                     global-dir (state/get-current-repo)
+            repo (cond
+                   global-dir repo
                  ;; FIXME(andelf): hack for demo graph, demo graph does not bind to local directory
-                     (string/starts-with? dir "memory://") "Logseq demo"
-                     :else (config/get-local-repo dir))
-              repo-dir (config/get-local-dir repo)
-              {:keys [mtime]} stat
-              ext (keyword (path/file-ext path))]
-          (when (contains? #{:org :md :markdown :css :js :edn :excalidraw :tldr} ext)
-            (p/let [db-content (db-async/<get-file repo path)
-                    exists-in-db? (not (nil? db-content))
-                    db-content (or db-content "")]
-              (when (or content (contains? #{"unlink" "unlinkDir" "addDir"} type))
-                (cond
-                  (and (= "unlinkDir" type) dir)
-                  (state/pub-event! [:graph/dir-gone dir])
-
-                  (and (= "addDir" type) dir)
-                  (state/pub-event! [:graph/dir-back repo dir])
-
-                  (contains? (:file/unlinked-dirs @state/state) dir)
-                  nil
-
-                  (and (= "add" type)
-                       (not= (string/trim content) (string/trim db-content)))
-                  (let [backup? (not (string/blank? db-content))]
-                    (handle-add-and-change! repo path content db-content mtime backup?))
-
-                  (and (= "change" type)
-                       (= dir repo-dir)
-                       (not= (string/trim content) (string/trim db-content))
-                       (not (common-config/local-asset? path)))
-                  (when-not (and
-                             (string/includes? path (str "/" (config/get-journals-directory) "/"))
-                             (or
-                              (= (string/trim content)
-                                 (string/trim (or (state/get-default-journal-template) "")))
-                              (= (string/trim content) "-")
-                              (= (string/trim content) "*")))
-                    (handle-add-and-change! repo path content db-content mtime (not global-dir))) ;; no backup for global dir
-
-                  (and (= "unlink" type)
-                       exists-in-db?)
-                  (p/let [dir-exists? (fs/file-exists? dir "")]
-                    (when dir-exists?
-                      (when-let [page-name (db/get-file-page path)]
-                        (println "Delete page: " page-name ", file path: " path ".")
-                        (page-handler/<delete! page-name #()))))
+                   (string/starts-with? dir "memory://") "Logseq demo"
+                   :else (config/get-local-repo dir))
+            repo-dir (config/get-local-dir repo)
+            {:keys [mtime]} stat
+            ext (keyword (path/file-ext path))]
+        (when (contains? #{:org :md :markdown :css :js :edn :excalidraw :tldr} ext)
+          (p/let [db-content (db-async/<get-file repo path)
+                  exists-in-db? (not (nil? db-content))
+                  db-content (or db-content "")]
+            (when (or content (contains? #{"unlink" "unlinkDir" "addDir"} type))
+              (cond
+                (and (= "unlinkDir" type) dir)
+                (state/pub-event! [:graph/dir-gone dir])
+
+                (and (= "addDir" type) dir)
+                (state/pub-event! [:graph/dir-back repo dir])
+
+                (contains? (:file/unlinked-dirs @state/state) dir)
+                nil
+
+                (and (= "add" type)
+                     (not= (string/trim content) (string/trim db-content)))
+                (let [backup? (not (string/blank? db-content))]
+                  (handle-add-and-change! repo path content db-content mtime backup?))
+
+                (and (= "change" type)
+                     (= dir repo-dir)
+                     (not (common-config/local-asset? path)))
+                (handle-add-and-change! repo path content db-content mtime (not global-dir)) ;; no backup for global dir
+
+                (and (= "unlink" type)
+                     exists-in-db?)
+                (p/let [dir-exists? (fs/file-exists? dir "")]
+                  (when dir-exists?
+                    (when-let [page-name (db/get-file-page path)]
+                      (println "Delete page: " page-name ", file path: " path ".")
+                      (page-handler/<delete! page-name #()))))
 
           ;; global config handling
-                  (and (= "change" type)
-                       (= dir (global-config-handler/global-config-dir)))
-                  (when (= path "config.edn")
-                    (file-handler/alter-global-file
-                     (global-config-handler/global-config-path) content {:from-disk? true}))
+                (and (= "change" type)
+                     (= dir (global-config-handler/global-config-dir)))
+                (when (= path "config.edn")
+                  (file-handler/alter-global-file
+                   (global-config-handler/global-config-path) content {:from-disk? true}))
 
-                  (and (= "change" type)
-                       (not exists-in-db?))
-                  (js/console.error "Can't get file in the db: " path)
+                (and (= "change" type)
+                     (not exists-in-db?))
+                (js/console.error "Can't get file in the db: " path)
 
-                  (and (contains? #{"add" "change" "unlink"} type)
-                       (string/ends-with? path "logseq/custom.css"))
-                  (do
-                    (println "reloading custom.css")
-                    (ui-handler/add-style-if-exists!))
+                (and (contains? #{"add" "change" "unlink"} type)
+                     (string/ends-with? path "logseq/custom.css"))
+                (do
+                  (println "reloading custom.css")
+                  (ui-handler/add-style-if-exists!))
 
-                  (contains? #{"add" "change" "unlink"} type)
-                  nil
+                (contains? #{"add" "change" "unlink"} type)
+                nil
 
-                  :else
-                  (log/error :fs/watcher-no-handler {:type type
-                                                     :payload payload})))))
+                :else
+                (log/error :fs/watcher-no-handler {:type type
+                                                   :payload payload})))))
 
       ;; return nil, otherwise the entire db will be transferred by ipc
-          nil)))))
+        nil))))
 
 (defn load-graph-files!
   "This fn replaces the former initial fs watcher"

src/main/frontend/handler/common/file.cljs

@@ -82,13 +82,6 @@
       :fs/local-file-change
       (graph-parser/parse-file db-conn file-path content (assoc-in options [:extract-options :resolve-uuid-fn] diff-merge-uuids-2ways))
 
-      ;; TODO Junyi: 3 ways to handle remote file change
-      ;; The file is on remote, so we should have
-      ;;   1. a "common ancestor" file locally
-      ;;     the worst case is that the file is not in db, so we should use the
-      ;;     empty file as the common ancestor
-      ;;   2. a "remote version" just fetched from remote
-
       ;; default to parse the file
       (graph-parser/parse-file db-conn file-path content options))))
 

src/main/frontend/handler/events.cljs

@@ -394,18 +394,10 @@
   (p/let [_ (page-handler/create-today-journal!)]
     (ui-handler/re-render-root!)))
 
-;; FIXME: this still happens when writing fast
 (defmethod handle :file/not-matched-from-disk [[_ path disk-content db-content]]
   (when-let [repo (state/get-current-repo)]
-    (let [^js sqlite @db-browser/*worker]
-      (p/let [writes-finished? (when sqlite (.file-writes-finished? sqlite (state/get-current-repo)))
-              request-finished? (db-transact/request-finished?)]
-        (prn :debug :writes-finished? writes-finished?
-          :request-finished? request-finished?)
-        (when (and request-finished? writes-finished? disk-content db-content
-                (not= (util/trim-safe disk-content) (util/trim-safe db-content)))
-          (state/set-modal! #(diff/local-file repo path disk-content db-content)
-            {:label "diff__cp"}))))))
+    (state/set-modal! #(diff/local-file repo path disk-content db-content)
+                      {:label "diff__cp"})))
 
 
 (defmethod handle :modal/display-file-version-selector  [[_ versions path  get-content]]

src/main/frontend/modules/outliner/ui.cljc

@@ -9,29 +9,28 @@
 (defmacro transact!
   [opts & body]
   `(let [test?# frontend.util/node-test?]
-     (when (or test?# (frontend.db.transact/request-finished?))
-       (let [ops# frontend.modules.outliner.op/*outliner-ops*
-             editor-info# (state/get-editor-info)]
-         (if ops#
-           (do ~@body)                    ; nested transact!
-           (binding [frontend.modules.outliner.op/*outliner-ops* (transient [])]
-             ~@body
-             (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*)
-                   worker# @state/*db-worker]
-               (if test?#
-                 (when (seq r#)
-                   (logseq.outliner.op/apply-ops! (state/get-current-repo)
-                                                  (frontend.db.conn/get-db false)
-                                                  r#
-                                                  (state/get-date-formatter)
-                                                  ~opts))
-                 (when (and worker# (seq r#))
-                   (let [request-id# (state/get-worker-next-request-id)
-                         request# #(.apply-outliner-ops ^Object worker# (state/get-current-repo)
-                                                        (pr-str r#)
-                                                        (pr-str (assoc ~opts
-                                                                       :request-id request-id#
-                                                                       :editor-info editor-info#)))
-                         response# (state/add-worker-request! request-id# request#)]
+     (let [ops# frontend.modules.outliner.op/*outliner-ops*
+           editor-info# (state/get-editor-info)]
+       (if ops#
+         (do ~@body)                    ; nested transact!
+         (binding [frontend.modules.outliner.op/*outliner-ops* (transient [])]
+           ~@body
+           (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*)
+                 worker# @state/*db-worker]
+             (if test?#
+               (when (seq r#)
+                 (logseq.outliner.op/apply-ops! (state/get-current-repo)
+                                                (frontend.db.conn/get-db false)
+                                                r#
+                                                (state/get-date-formatter)
+                                                ~opts))
+               (when (and worker# (seq r#))
+                 (let [request-id# (state/get-worker-next-request-id)
+                       request# #(.apply-outliner-ops ^Object worker# (state/get-current-repo)
+                                                      (pr-str r#)
+                                                      (pr-str (assoc ~opts
+                                                                     :request-id request-id#
+                                                                     :editor-info editor-info#)))
+                       response# (state/add-worker-request! request-id# request#)]
 
-                     response#))))))))))
+                   response#)))))))))