wip: move file parse to worker

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -51,7 +51,7 @@
   (let [existing-file-page (get-file-page db file-path)
         pages-to-clear (distinct (filter some? [existing-file-page (:db/id file-page)]))
         blocks (mapcat (fn [page-id]
-                         (ldb/get-page-blocks db page-id {:pull-keys [:db/id :block/uuid]}))
+                         (:block/_page (d/entity db page-id)))
                        pages-to-clear)
         retain-uuids (set (keep :block/uuid retain-uuid-blocks))]
     (retract-blocks-tx (distinct blocks) retain-uuids)))
@@ -63,16 +63,14 @@ Options available:
   * :delete-blocks-fn - Optional fn which is called with the new page, file and existing block uuids
   which may be referenced elsewhere. Used to delete the existing blocks before saving the new ones.
    Implemented in file-common-handler/validate-and-get-blocks-to-delete for IoC
-* :skip-db-transact? - Boolean which skips transacting in order to batch transactions. Default is false
-* :extract-options - Options map to pass to extract/extract"
+  * :extract-options - Options map to pass to extract/extract"
   ([conn file-path content] (parse-file conn file-path content {}))
-  ([conn file-path content {:keys [delete-blocks-fn extract-options skip-db-transact? ctime mtime]
-                            :or {delete-blocks-fn (constantly [])
-                                 skip-db-transact? false}
+  ([conn file-path content {:keys [delete-blocks-fn extract-options ctime mtime]
+                            :or {delete-blocks-fn (constantly [])}
                             :as options}]
    (let [format (common-util/get-format file-path)
          file-content [{:file/path file-path}]
-         {:keys [tx ast]}
+         {:keys [tx]}
          (let [extract-options' (merge {:block-pattern (common-config/get-block-pattern format)
                                         :date-formatter "MMM do, yyyy"
                                         :uri-encoded? false
@@ -91,7 +89,7 @@ Options available:
 
                      :else nil)
                block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
-               delete-blocks (delete-blocks-fn @conn (first pages) file-path block-ids)
+               delete-blocks (delete-blocks-fn (first pages) file-path block-ids)
                block-refs-ids (->> (mapcat :block/refs blocks)
                                    (filter (fn [ref] (and (vector? ref)
                                                           (= :block/uuid (first ref)))))
@@ -110,14 +108,8 @@ Options available:
                           (or ctime (nil? file-entity))
                           (assoc :file/created-at (or ctime (js/Date.)))
                           mtime
-                          (assoc :file/last-modified-at mtime))])
-         result (if skip-db-transact?
-                  tx
-                  (do
-                    (ldb/transact! conn tx (select-keys options [:new-graph? :from-disk?]))
-                    nil))]
-     {:tx result
-      :ast ast})))
+                          (assoc :file/last-modified-at mtime))])]
+     (ldb/transact! conn tx (select-keys options [:new-graph? :from-disk?])))))
 
 (defn filter-files
   "Filters files in preparation for parsing. Only includes files that are

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -698,23 +698,16 @@
                                                    (str (gp-property/colons-org "id") " " (:block/uuid block)))))]
                                (string/replace-first c replace-str ""))))))
 
-(defn block-exists-in-another-page?
-  "For sanity check only.
-   For renaming file externally, the file is actually deleted and transacted before-hand."
-  [db block-uuid current-page-name]
-  (when (and db current-page-name)
-    (when-let [block-page-name (:block/name (:block/page (d/entity db [:block/uuid block-uuid])))]
-      (not= current-page-name block-page-name))))
-
 (defn fix-block-id-if-duplicated!
-  "If the block exists in another page, we need to fix it
-   If the block exists in the current extraction process, we also need to fix it"
-  [db page-name *block-exists-in-extraction block]
-  (let [block (if (or (@*block-exists-in-extraction (:block/uuid block))
-                      (block-exists-in-another-page? db (:block/uuid block) page-name))
+  "If the block exists in another page or the current page, we need to fix it"
+  [db page-name *extracted-block-ids block]
+  (let [existing-block (d/entity db [:block/uuid (:block/uuid block)])
+        block (if (and existing-block
+                       (or (not= (:block/name (:block/page existing-block)) page-name)
+                           (contains? @*extracted-block-ids (:block/uuid block))))
                 (fix-duplicate-id block)
                 block)]
-    (swap! *block-exists-in-extraction conj (:block/uuid block))
+    (swap! *extracted-block-ids conj (:block/uuid block))
     block))
 
 (defn extract-blocks

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -3,10 +3,10 @@
   (:require ["fs" :as fs]
             ["path" :as path]
             [clojure.edn :as edn]
-            [logseq.common.graph :as common-graph]
             [logseq.common.config :as common-config]
-            [logseq.graph-parser :as graph-parser]
+            [logseq.common.graph :as common-graph]
             [logseq.common.util :as common-util]
+            [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.db :as gp-db]))
 
 (defn- slurp
@@ -28,10 +28,10 @@
   [dir* config]
   (let [dir (path/resolve dir*)]
     (->> (common-graph/get-files dir)
-        (map #(hash-map :file/path %))
-        graph-parser/filter-files
-        (remove-hidden-files dir config)
-        (mapv #(assoc % :file/content (slurp (:file/path %)))))))
+         (map #(hash-map :file/path %))
+         graph-parser/filter-files
+         (remove-hidden-files dir config)
+         (mapv #(assoc % :file/content (slurp (:file/path %)))))))
 
 (defn- read-config
   "Reads repo-specific config from logseq/config.edn"
@@ -45,8 +45,7 @@
   [conn files {:keys [config] :as options}]
   (let [extract-options (merge {:date-formatter (common-config/get-date-formatter config)
                                 :user-config config
-                                :filename-format (or (:file/name-format config) :legacy)
-                                :extracted-block-ids (atom #{})}
+                                :filename-format (or (:file/name-format config) :legacy)}
                                (select-keys options [:verbose]))]
     (mapv
      (fn [{:file/keys [path content]}]

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -220,15 +220,13 @@
 (defn- extract-pages-and-blocks
   "uri-encoded? - if is true, apply URL decode on the file path
    options -
-     :extracted-block-ids - An atom that contains all block ids that have been extracted in the current page (not yet saved to db)
      :resolve-uuid-fn - Optional fn which is called to resolve uuids of each block. Enables diff-merge
        (2 ways diff) based uuid resolution upon external editing.
        returns a list of the uuids, given the receiving ast, or nil if not able to resolve.
        Implemented in reset-file-handler/diff-merge-uuids-2ways for IoC
        Called in gp-extract/extract as AST is being parsed and properties are extracted there"
-  [format ast properties file content {:keys [date-formatter db filename-format extracted-block-ids resolve-uuid-fn]
-                                       :or {extracted-block-ids (atom #{})
-                                            resolve-uuid-fn (constantly nil)}
+  [format ast properties file content {:keys [date-formatter db filename-format resolve-uuid-fn]
+                                       :or {resolve-uuid-fn (constantly nil)}
                                        :as options}]
   (assert db "Datascript DB is required")
   (try
@@ -237,9 +235,10 @@
           options' (assoc options :page-name page-name)
           ;; In case of diff-merge (2way) triggered, use the uuids to override the ones extracted from the AST
           override-uuids (resolve-uuid-fn format ast content options')
+          *extracted-block-ids (atom #{})
           blocks (->> (gp-block/extract-blocks ast content format options')
                       (attach-block-ids-if-match override-uuids)
-                      (mapv #(gp-block/fix-block-id-if-duplicated! db page-name extracted-block-ids %))
+                      (mapv #(gp-block/fix-block-id-if-duplicated! db page-name *extracted-block-ids %))
                       ;; FIXME: use page uuid
                       (gp-block/with-parent-and-order {:block/name page-name})
                       (vec))

deps/graph-parser/test/logseq/graph_parser_test.cljs

@@ -1,12 +1,12 @@
 (ns logseq.graph-parser-test
   (:require [cljs.test :refer [deftest testing is are]]
             [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db :as ldb]
             [logseq.graph-parser :as graph-parser]
-            [logseq.graph-parser.db :as gp-db]
             [logseq.graph-parser.block :as gp-block]
-            [logseq.graph-parser.property :as gp-property]
-            [datascript.core :as d]
-            [logseq.db :as ldb]))
+            [logseq.graph-parser.db :as gp-db]
+            [logseq.graph-parser.property :as gp-property]))
 
 (def foo-edn
   "Example exported whiteboard page as an edn exportable."
@@ -78,7 +78,7 @@
                                                         (throw (js/Error "Testing unexpected failure")))]
         (try
           (parse-file conn "foo.md" "- id:: 628953c1-8d75-49fe-a648-f4c612109098"
-                      {:delete-blocks-fn (fn [_db page _file _uuids]
+                      {:delete-blocks-fn (fn [page _file _uuids]
                                            (reset! deleted-page page))})
           (catch :default _)))
       (is (= nil @deleted-page)

src/main/frontend/handler/file_based/file.cljs

@@ -7,7 +7,6 @@
             [frontend.db.file-based.model :as file-model]
             [frontend.fs :as fs]
             [frontend.handler.common.config-edn :as config-edn-common-handler]
-            [frontend.handler.file-based.reset-file :as reset-file-handler]
             [frontend.handler.global-config :as global-config-handler]
             [frontend.handler.repo-config :as repo-config-handler]
             [frontend.handler.ui :as ui-handler]
@@ -33,6 +32,10 @@
       (println "Load file failed: " path)
       (js/console.error e)))))
 
+(defn reset-file!
+  [repo file-path content opts]
+  (state/<invoke-db-worker :thread-api/reset-file repo file-path content opts))
+
 (defn- load-multiple-files
   [repo-url paths]
   (doall
@@ -133,7 +136,7 @@
 (defn alter-file
   "Write any in-DB file, e.g. repo config, page, whiteboard, etc."
   [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose
-                             skip-db-transact? extracted-block-ids ctime mtime]
+                             ctime mtime]
                       :fs/keys [event]
                       :or {reset? true
                            re-render-root? false
@@ -147,25 +150,22 @@
     (when (or config-valid? (not config-file?)) ; non-config file or valid config
       (let [opts {:new-graph? new-graph?
                   :from-disk? from-disk?
-                  :skip-db-transact? skip-db-transact?
                   :fs/event event
                   :ctime ctime
-                  :mtime mtime}
-            result (if reset?
-                     (do
-                       (when-not skip-db-transact?
-                         (when-let [page-id (file-model/get-file-page-id path)]
-                           (db/transact! repo
-                                         [[:db/retract page-id :block/alias]
-                                          [:db/retract page-id :block/tags]]
-                                         opts)))
-                       (reset-file-handler/reset-file!
-                        repo path content (merge opts
-                                                 ;; To avoid skipping the `:or` bounds for keyword destructuring
-                                                 (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids})
-                                                 (when (some? verbose) {:verbose verbose}))))
-                     (db/set-file-content! repo path content opts))]
-        (-> (p/let [_ (when-not from-disk?
+                  :mtime mtime}]
+        (-> (p/let [result (if reset?
+                             (p/do!
+                              (when-let [page-id (file-model/get-file-page-id path)]
+                                (db/transact! repo
+                                              [[:db/retract page-id :block/alias]
+                                               [:db/retract page-id :block/tags]]
+                                              opts))
+                              (reset-file!
+                               repo path content (merge opts
+                                                         ;; To avoid skipping the `:or` bounds for keyword destructuring
+                                                        (when (some? verbose) {:verbose verbose}))))
+                             (db/set-file-content! repo path content opts))
+                    _ (when-not from-disk?
                         (write-file-aux! repo path content {:skip-compare? skip-compare?}))]
               (when re-render-root? (ui-handler/re-render-root!))
 
@@ -178,15 +178,16 @@
 
                 (= path "logseq/config.edn")
                 (p/let [_ (repo-config-handler/restore-repo-config! repo content)]
-                  (state/pub-event! [:shortcut/refresh]))))
+                  (state/pub-event! [:shortcut/refresh])))
+
+              result)
             (p/catch
              (fn [error]
                (println "Write file failed, path: " path ", content: " content)
                (log/error :write/failed error)
                (state/pub-event! [:capture-error
                                   {:error error
-                                   :payload {:type :write-file/failed-for-alter-file}}]))))
-        result))))
+                                   :payload {:type :write-file/failed-for-alter-file}}]))))))))
 
 (defn set-file-content!
   [repo path new-content]
@@ -234,10 +235,13 @@
                                 (map (fn [path] (db/get-file repo path)) paths)))]
     ;; update db
     (when update-db?
-      (doseq [[path content] files]
-        (if reset?
-          (reset-file-handler/reset-file! repo path content)
-          (db/set-file-content! repo path content))))
+      (p/all
+       (map
+        (fn [[path content]]
+          (if reset?
+            (reset-file! repo path content {})
+            (db/set-file-content! repo path content)))
+        files)))
     (alter-files-handler! repo files opts file->content)))
 
 (defn watch-for-current-graph-dir!

src/main/frontend/handler/file_based/repo.cljs

@@ -7,7 +7,6 @@
             [frontend.db.file-based.model :as file-model]
             [frontend.fs :as fs]
             [frontend.handler.file-based.file :as file-handler]
-            [frontend.handler.file-based.reset-file :as reset-file-handler]
             [frontend.handler.repo-config :as repo-config-handler]
             [frontend.handler.route :as route-handler]
             [frontend.handler.ui :as ui-handler]
@@ -38,7 +37,7 @@
         (p/let [_ (fs/mkdir-if-not-exists (path/path-join repo-dir pages-dir))
                 file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath default-content)]
           (when-not file-exists?
-            (reset-file-handler/reset-file! repo-url file-rpath default-content)))))))
+            (file-handler/reset-file! repo-url file-rpath default-content {})))))))
 
 (defn- create-custom-theme
   [repo-url]
@@ -50,7 +49,7 @@
     (p/let [_ (fs/mkdir-if-not-exists (path/path-join repo-dir config/app-name))
             file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath default-content)]
       (when-not file-exists?
-        (reset-file-handler/reset-file! repo-url path default-content)))))
+        (file-handler/reset-file! repo-url path default-content {})))))
 
 (comment
   (defn- create-dummy-notes-page
@@ -60,7 +59,7 @@
           file-rpath (str (config/get-pages-directory) "/how_to_make_dummy_notes.md")]
       (p/let [_ (fs/mkdir-if-not-exists (path/path-join repo-dir (config/get-pages-directory)))
               _file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath content)]
-        (reset-file-handler/reset-file! repo-url file-rpath content)))))
+        (file-handler/reset-file! repo-url file-rpath content {})))))
 
 (defn create-config-file-if-not-exists
   "Creates a default logseq/config.edn if it doesn't exist"
@@ -74,7 +73,7 @@
             path (str app-dir "/" config/config-file)]
         (p/let [file-exists? (fs/create-if-not-exists repo-url repo-dir "logseq/config.edn" default-content)]
           (when-not file-exists?
-            (reset-file-handler/reset-file! repo-url path default-content)
+            (file-handler/reset-file! repo-url path default-content {})
             (repo-config-handler/set-repo-config-state! repo-url default-content)))))))
 
 (defn- create-default-files!
@@ -89,36 +88,30 @@
            (create-custom-theme repo-url)
            (state/pub-event! [:page/create-today-journal repo-url]))))
 
-(defonce *file-tx (atom nil))
-
 (defn- parse-and-load-file!
   "Accept: .md, .org, .edn, .css"
-  [repo-url file {:keys [new-graph? verbose skip-db-transact? extracted-block-ids]
-                  :or {skip-db-transact? true}}]
-  (try
-    (reset! *file-tx
-            (file-handler/alter-file repo-url
-                                     (:file/path file)
-                                     (:file/content file)
-                                     (merge (:stat file)
-                                            {:new-graph? new-graph?
-                                             :re-render-root? false
-                                             :from-disk? true
-                                             :skip-db-transact? skip-db-transact?}
-                                            ;; To avoid skipping the `:or` bounds for keyword destructuring
-                                            (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids})
-                                            (when (some? verbose) {:verbose verbose}))))
-    (state/set-parsing-state! (fn [m]
-                                (update m :finished inc)))
-    @*file-tx
-    (catch :default e
-      (println "Parse and load file failed: " (str (:file/path file)))
-      (js/console.error e)
-      (state/set-parsing-state! (fn [m]
-                                  (update m :failed-parsing-files conj [(:file/path file) e])))
-      (state/set-parsing-state! (fn [m]
-                                  (update m :finished inc)))
-      nil)))
+  [repo-url file {:keys [new-graph? verbose]}]
+  (->
+   (p/let [result (file-handler/alter-file repo-url
+                                           (:file/path file)
+                                           (:file/content file)
+                                           (merge (:stat file)
+                                                  {:new-graph? new-graph?
+                                                   :re-render-root? false
+                                                   :from-disk? true}
+                                                  ;; To avoid skipping the `:or` bounds for keyword destructuring
+                                                  (when (some? verbose) {:verbose verbose})))]
+     (state/set-parsing-state! (fn [m]
+                                 (update m :finished inc)))
+     result)
+   (p/catch (fn [e]
+              (println "Parse and load file failed: " (str (:file/path file)))
+              (js/console.error e)
+              (state/set-parsing-state! (fn [m]
+                                          (update m :failed-parsing-files conj [(:file/path file) e])))
+              (state/set-parsing-state! (fn [m]
+                                          (update m :finished inc)))
+              nil))))
 
 (defn- after-parse
   [repo-url re-render? re-render-opts opts graph-added-chan]
@@ -145,8 +138,7 @@
         total (count supported-files)
         large-graph? (> total 1000)
         *page-names (atom #{})
-        *page-name->path (atom {})
-        *extracted-block-ids (atom #{})]
+        *page-name->path (atom {})]
     (when (seq delete-data) (db/transact! repo-url delete-data {:delete-files? true}))
     (state/set-current-repo! repo-url)
     (state/set-parsing-state! {:total (count supported-files)})
@@ -157,11 +149,9 @@
           (state/set-parsing-state! (fn [m]
                                       (assoc m
                                              :current-parsing-file (:file/path file))))
-          (parse-and-load-file! repo-url file (assoc
-                                               (select-keys opts [:new-graph? :verbose])
-                                               :skip-db-transact? false)))
+          (parse-and-load-file! repo-url file (select-keys opts [:new-graph? :verbose])))
         (after-parse repo-url re-render? re-render-opts opts graph-added-chan))
-      (async/go-loop [tx []]
+      (async/go-loop []
         (if-let [item (async/<! chan)]
           (let [[idx file] item
                 whiteboard? (common-config/whiteboard? (:file/path file))
@@ -174,13 +164,9 @@
 
             (when yield-for-ui? (async/<! (async/timeout 1)))
 
-            (let [opts' (-> (select-keys opts [:new-graph? :verbose])
-                            (assoc :extracted-block-ids *extracted-block-ids))
+            (let [opts' (select-keys opts [:new-graph? :verbose])
                   ;; whiteboards might have conflicting block IDs so that db transaction could be failed
-                  opts' (if whiteboard?
-                          (assoc opts' :skip-db-transact? false)
-                          opts')
-                  result (parse-and-load-file! repo-url file opts')
+                  result (async/<! (p->c (parse-and-load-file! repo-url file opts')))
                   page-name (when (coll? result) ; result could be a promise
                               (some (fn [x] (when (and (map? x)
                                                        (:block/title x)
@@ -188,29 +174,19 @@
                                               (:block/name x)))
                                     result))
                   page-exists? (and page-name (get @*page-names page-name))
-                  tx' (cond
-                        whiteboard? tx
-                        page-exists? (do
-                                       (state/pub-event! [:notification/show
-                                                          {:content [:div
-                                                                     (util/format "The file \"%s\" will be skipped because another file \"%s\" has the same page title."
-                                                                                  (:file/path file)
-                                                                                  (get @*page-name->path page-name))]
-                                                           :status :warning
-                                                           :clear? false}])
-                                       tx)
-                        :else (concat tx result))
+                  _ (when page-exists?
+                      (state/pub-event! [:notification/show
+                                         {:content [:div
+                                                    (util/format "The file \"%s\" will be skipped because another file \"%s\" has the same page title."
+                                                                 (:file/path file)
+                                                                 (get @*page-name->path page-name))]
+                                          :status :warning
+                                          :clear? false}]))
                   _ (when (and page-name (not page-exists?))
                       (swap! *page-names conj page-name)
-                      (swap! *page-name->path assoc page-name (:file/path file)))
-                  tx' (if (zero? (rem (inc idx) 100))
-                        (do
-                          (async/<! (p->c (db/transact! repo-url tx' {:from-disk? true})))
-                          [])
-                        tx')]
-              (recur tx')))
+                      (swap! *page-name->path assoc page-name (:file/path file)))]
+              (recur)))
           (p/do!
-           (when (seq tx) (db/transact! repo-url tx {:from-disk? true}))
            (after-parse repo-url re-render? re-render-opts opts graph-added-chan)))))
     graph-added-chan))
 

src/main/frontend/handler/file_based/reset_file.cljs

@@ -1,105 +0,0 @@
-(ns frontend.handler.file-based.reset-file
-  "Fns for resetting a db file with parsed file content"
-  (:require [frontend.config :as config]
-            [frontend.state :as state]
-            [frontend.db :as db]
-            [frontend.db.file-based.model :as file-model]
-            [logseq.graph-parser :as graph-parser]
-            [logseq.common.util :as common-util]
-            [frontend.fs.diff-merge :as diff-merge]
-            [frontend.fs :as fs]
-            [frontend.context.i18n :refer [t]]
-            [promesa.core :as p]
-            [clojure.string :as string]
-            [cljs-bean.core :as bean]
-            [lambdaisland.glogi :as log]))
-
-(defn- page-exists-in-another-file
-  "Conflict of files towards same page"
-  [repo-url page file]
-  (when-let [page-name (:block/name page)]
-    (let [current-file (:file/path (file-model/get-page-file repo-url page-name))]
-      (when (not= file current-file)
-        current-file))))
-
-(defn- validate-existing-file
-  "Handle the case when the file is already exists in db
-     Likely caused by renaming between caps and non-caps, then cause file system
-     bugs on some OS
-     e.g. on macOS, it doesn't fire the file change event when renaming between
-       caps and non-caps"
-  [repo-url file-page file-path]
-  (when-let [current-file (page-exists-in-another-file repo-url file-page file-path)]
-    (when (not= file-path current-file)
-      (cond
-        ;; TODO: handle case sensitive file system
-        (= (common-util/path-normalize (string/lower-case current-file))
-           (common-util/path-normalize (string/lower-case file-path)))
-        ;; case renamed
-        (when-let [file (db/pull [:file/path current-file])]
-          (p/let [disk-content (fs/read-file "" current-file)]
-            (fs/backup-db-file! repo-url current-file (:file/content file) disk-content))
-          (db/transact! repo-url [{:db/id (:db/id file)
-                                   :file/path file-path}]))
-
-        :else
-        (let [error (t :file/validate-existing-file-error current-file file-path)]
-          (state/pub-event! [:notification/show
-                             {:content error
-                              :status :error
-                              :clear? false}]))))))
-
-(defn- validate-and-get-blocks-to-delete
-  "An implementation for the delete-blocks-fn in graph-parser/parse-file"
-  [repo-url db file-page file-path retain-uuid-blocks]
-  (validate-existing-file repo-url file-page file-path)
-  (graph-parser/get-blocks-to-delete db file-page file-path retain-uuid-blocks))
-
-(defn- diff-merge-uuids-2ways
-  "Infer new uuids from existing DB data and diff with the new AST
-   Return a list of uuids for the new blocks"
-  [format ast content {:keys [page-name] :as options}]
-  (try
-    (let [base-diffblocks (diff-merge/db->diff-blocks page-name)
-          income-diffblocks (diff-merge/ast->diff-blocks ast content format options)
-          diff-ops (diff-merge/diff base-diffblocks income-diffblocks)
-          new-uuids (diff-merge/attachUUID diff-ops (map :uuid base-diffblocks))]
-      (bean/->clj new-uuids))
-    (catch js/Error e
-      (log/error :diff-merge/diff-merge-2way-calling-failed e))))
-
-(defn- reset-file!*
-  "Parse file considering diff-merge with local or remote file
-   Decide how to treat the parsed file based on the file's triggering event
-   options -
-     :fs/reset-event - the event that triggered the file update
-     :fs/local-file-change - file changed on local disk
-     :fs/remote-file-change - file changed on remote"
-  [repo-url file-path content {:fs/keys [event] :as options}]
-  (when-let [db-conn (db/get-db repo-url false)]
-    (case event
-      ;; the file is already in db, so we can use the existing file's blocks
-      ;; to do the diff-merge
-      :fs/local-file-change
-      (graph-parser/parse-file db-conn file-path content (assoc-in options [:extract-options :resolve-uuid-fn] diff-merge-uuids-2ways))
-
-      ;; default to parse the file
-      (graph-parser/parse-file db-conn file-path content options))))
-
-(defn reset-file!
-  "Main fn for updating a db with the results of a parsed file"
-  ([repo-url file-path content]
-   (reset-file! repo-url file-path content {}))
-  ([repo-url file-path content {:keys [verbose extracted-block-ids _ctime _mtime] :as options}]
-   (let [options (merge (dissoc options :verbose :extracted-block-ids)
-                        {:delete-blocks-fn (partial validate-and-get-blocks-to-delete repo-url)
-                         ;; Options here should also be present in gp-cli/parse-graph
-                         :extract-options (merge
-                                           {:user-config (state/get-config)
-                                            :date-formatter (state/get-date-formatter)
-                                            :block-pattern (config/get-block-pattern (common-util/get-format file-path))
-                                            :filename-format (state/get-filename-format repo-url)}
-                                           ;; To avoid skipping the `:or` bounds for keyword destructuring
-                                           (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids})
-                                           (when (some? verbose) {:verbose verbose}))})]
-     (:tx (reset-file!* repo-url file-path content options)))))

src/main/frontend/worker/db_worker.cljs

@@ -20,6 +20,7 @@
             [frontend.worker.db.validate :as worker-db-validate]
             [frontend.worker.export :as worker-export]
             [frontend.worker.file :as file]
+            [frontend.worker.file.reset :as file-reset]
             [frontend.worker.handler.page :as worker-page]
             [frontend.worker.handler.page.file-based.rename :as file-worker-page-rename]
             [frontend.worker.rtc.asset-db-listener]
@@ -782,6 +783,12 @@
   [graph]
   (fix-broken-graph graph))
 
+(def-thread-api :thread-api/reset-file
+  [repo file-path content opts]
+  ;; (prn :debug :reset-file :file-path file-path :opts opts)
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (file-reset/reset-file! repo conn file-path content opts)))
+
 (comment
   (def-thread-api :general/dangerousRemoveAllDbs
     []

src/main/frontend/worker/file/reset.cljs

@@ -0,0 +1,83 @@
+(ns frontend.worker.file.reset
+  "Fns for resetting a db file with parsed file content"
+  (:require [clojure.string :as string]
+            [datascript.core :as d]
+            [frontend.worker.state :as worker-state]
+            [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
+            [logseq.db :as ldb]
+            [logseq.graph-parser :as graph-parser]
+            [logseq.graph-parser.db :as gp-db]))
+
+(defn- page-exists-in-another-file
+  "Conflict of files towards same page"
+  [db page file]
+  (when-let [page-name (:block/name page)]
+    (let [current-file (:file/path (gp-db/get-page-file db page-name))]
+      (when (not= file current-file)
+        current-file))))
+
+(defn- validate-existing-file
+  "Handle the case when the file is already exists in db
+     Likely caused by renaming between caps and non-caps, then cause file system
+     bugs on some OS
+     e.g. on macOS, it doesn't fire the file change event when renaming between
+       caps and non-caps"
+  [repo conn file-page file-path]
+  (when-let [current-file (page-exists-in-another-file @conn file-page file-path)]
+    (let [db @conn]
+      (when (not= file-path current-file)
+        (cond
+        ;; TODO: handle case sensitive file system
+          (= (common-util/path-normalize (string/lower-case current-file))
+             (common-util/path-normalize (string/lower-case file-path)))
+        ;; case renamed
+          (when-let [file (d/entity @conn [:file/path current-file])]
+          ;; (p/let [disk-content (fs/read-file "" current-file)]
+            ;;   (fs/backup-db-file! repo current-file (:file/content file) disk-content))
+            (ldb/transact! conn [{:db/id (:db/id file)
+                                  :file/path file-path}]))
+
+          :else
+          nil
+          ;; (let [error (t :file/validate-existing-file-error current-file file-path)]
+          ;;   (state/pub-event! [:notification/show
+          ;;                      {:content error
+          ;;                       :status :error
+          ;;                       :clear? false}]))
+          )))))
+
+(defn- validate-and-get-blocks-to-delete
+  "An implementation for the delete-blocks-fn in graph-parser/parse-file"
+  [repo conn file-page file-path retain-uuid-blocks]
+  (validate-existing-file repo conn file-page file-path)
+  (graph-parser/get-blocks-to-delete @conn file-page file-path retain-uuid-blocks))
+
+(defn- reset-file!*
+  "Parse file.
+   Decide how to treat the parsed file based on the file's triggering event
+   options -
+     :fs/reset-event - the event that triggered the file update
+     :fs/local-file-change - file changed on local disk
+     :fs/remote-file-change - file changed on remote"
+  [db-conn file-path content options]
+  (graph-parser/parse-file db-conn file-path content options))
+
+(defn reset-file!
+  "Main fn for updating a db with the results of a parsed file"
+  ([repo conn file-path content]
+   (reset-file! repo conn file-path content {}))
+  ([repo conn file-path content {:keys [verbose _ctime _mtime] :as options}]
+   (let [config (worker-state/get-config repo)
+         options (merge (dissoc options :verbose)
+                        {:delete-blocks-fn (partial validate-and-get-blocks-to-delete repo conn)
+                         ;; Options here should also be present in gp-cli/parse-graph
+                         :extract-options (merge
+                                           {:user-config config
+                                            :date-formatter (worker-state/get-date-formatter repo)
+                                            :block-pattern (common-config/get-block-pattern
+                                                            (or (common-util/get-format file-path) :markdown))
+                                            :filename-format (:file/name-format config)}
+                                           ;; To avoid skipping the `:or` bounds for keyword destructuring
+                                           (when (some? verbose) {:verbose verbose}))})]
+     (:tx (reset-file!* conn file-path content options)))))

src/test/frontend/fs/diff_merge_test.cljs

@@ -3,7 +3,6 @@
             [cljs.test :refer [are deftest is]]
             [frontend.db.conn :as conn]
             [frontend.fs.diff-merge :as fs-diff]
-            [frontend.handler.file-based.reset-file :as reset-file-handler]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.db :as gp-db]
             [logseq.graph-parser.mldoc :as gp-mldoc]
@@ -25,13 +24,13 @@
   (are [text diff-blocks]
        (= (org-text->diffblocks text)
           diff-blocks)
-        ":PROPERTIES:
+    ":PROPERTIES:
 :ID:       72289d9a-eb2f-427b-ad97-b605a4b8c59b
 :END:
 #+tItLe: Well parsed!"
-[{:body ":PROPERTIES:\n:ID:       72289d9a-eb2f-427b-ad97-b605a4b8c59b\n:END:\n#+tItLe: Well parsed!"
-  :uuid "72289d9a-eb2f-427b-ad97-b605a4b8c59b"
-  :level 1}]
+    [{:body ":PROPERTIES:\n:ID:       72289d9a-eb2f-427b-ad97-b605a4b8c59b\n:END:\n#+tItLe: Well parsed!"
+      :uuid "72289d9a-eb2f-427b-ad97-b605a4b8c59b"
+      :level 1}]
 
     "#+title: Howdy"
     [{:body "#+title: Howdy" :uuid nil :level 1}]
@@ -62,37 +61,37 @@
   (are [text diff-blocks]
        (= (text->diffblocks text)
           diff-blocks)
-  "- a
+    "- a
 \t- b
 \t\t- c"
-  [{:body "a" :uuid nil :level 1}
-   {:body "b" :uuid nil :level 2}
-   {:body "c" :uuid nil :level 3}]
+    [{:body "a" :uuid nil :level 1}
+     {:body "b" :uuid nil :level 2}
+     {:body "c" :uuid nil :level 3}]
 
-"- a
+    "- a
 \t- b
 \t\t- c
 \t\t  multiline
 - d"
-[{:body "a" :uuid nil :level 1}
- {:body "b" :uuid nil :level 2}
- {:body "c\nmultiline" :uuid nil :level 3}
- {:body "d" :uuid nil :level 1}]
+    [{:body "a" :uuid nil :level 1}
+     {:body "b" :uuid nil :level 2}
+     {:body "c\nmultiline" :uuid nil :level 3}
+     {:body "d" :uuid nil :level 1}]
 
-  "## hello
+    "## hello
 \t- world
 \t\t- nice
 \t\t\t- nice
 \t\t\t- bingo
 \t\t\t- world"
-  [{:body "## hello" :uuid nil :level 1}
-   {:body "world" :uuid nil :level 2}
-   {:body "nice" :uuid nil :level 3}
-   {:body "nice" :uuid nil :level 4}
-   {:body "bingo" :uuid nil :level 4}
-   {:body "world" :uuid nil :level 4}]
-
-  "# a
+    [{:body "## hello" :uuid nil :level 1}
+     {:body "world" :uuid nil :level 2}
+     {:body "nice" :uuid nil :level 3}
+     {:body "nice" :uuid nil :level 4}
+     {:body "bingo" :uuid nil :level 4}
+     {:body "world" :uuid nil :level 4}]
+
+    "# a
 ## b
 ### c
 #### d
@@ -102,27 +101,27 @@
 \t\t- h
 \t- i
 - j"
-  [{:body "# a" :uuid nil :level 1}
-   {:body "## b" :uuid nil :level 1}
-   {:body "### c" :uuid nil :level 1}
-   {:body "#### d" :uuid nil :level 1}
-   {:body "### e" :uuid nil :level 1}
-   {:body "f" :uuid nil :level 1}
-   {:body "g" :uuid nil :level 2}
-   {:body "h" :uuid nil :level 3}
-   {:body "i" :uuid nil :level 2}
-   {:body "j" :uuid nil :level 1}]
+    [{:body "# a" :uuid nil :level 1}
+     {:body "## b" :uuid nil :level 1}
+     {:body "### c" :uuid nil :level 1}
+     {:body "#### d" :uuid nil :level 1}
+     {:body "### e" :uuid nil :level 1}
+     {:body "f" :uuid nil :level 1}
+     {:body "g" :uuid nil :level 2}
+     {:body "h" :uuid nil :level 3}
+     {:body "i" :uuid nil :level 2}
+     {:body "j" :uuid nil :level 1}]
 
     "- a\n  id:: 63e25526-3612-4fb1-8cf9-f66db1254a58
 \t- b
 \t\t- c"
-[{:body "a\nid:: 63e25526-3612-4fb1-8cf9-f66db1254a58"
-  :uuid "63e25526-3612-4fb1-8cf9-f66db1254a58" :level 1}
- {:body "b" :uuid nil :level 2}
- {:body "c" :uuid nil :level 3}]
+    [{:body "a\nid:: 63e25526-3612-4fb1-8cf9-f66db1254a58"
+      :uuid "63e25526-3612-4fb1-8cf9-f66db1254a58" :level 1}
+     {:body "b" :uuid nil :level 2}
+     {:body "c" :uuid nil :level 3}]
 
-  "alias:: ⭐️\nicon:: ⭐️"
-[{:body "alias:: ⭐️\nicon:: ⭐️", :level 1, :uuid nil}]))
+    "alias:: ⭐️\nicon:: ⭐️"
+    [{:body "alias:: ⭐️\nicon:: ⭐️", :level 1, :uuid nil}]))
 
 (defn text->diffblocks-alt
   [text]
@@ -206,7 +205,7 @@
 \t\t\t- nice
 \t\t\t- bingo
 \t\t\t- world"
-      "## Halooooo
+    "## Halooooo
 \t- world
 \t\t- nice
 \t\t\t- nice
@@ -216,26 +215,26 @@
     ;; See https://github.com/logseq/diff-merge#usage
     [[]
      [[-1 {:body "## hello"
-          :level 1
-          :uuid nil}]
+           :level 1
+           :uuid nil}]
       [1  {:body "## Halooooo"
-          :level 1
-          :uuid nil}]]
+           :level 1
+           :uuid nil}]]
      [[0 {:body "world"
-         :level 2
-         :uuid nil}]]
+          :level 2
+          :uuid nil}]]
      [[0 {:body "nice"
-         :level 3
-         :uuid nil}]]
+          :level 3
+          :uuid nil}]]
      [[0 {:body "nice"
-         :level 4
-         :uuid nil}]]
+          :level 4
+          :uuid nil}]]
      [[0 {:body "bingo"
-         :level 4
-         :uuid nil}]]
+          :level 4
+          :uuid nil}]]
      [[0 {:body "world"
-         :level 4
-         :uuid nil}]]]
+          :level 4
+          :uuid nil}]]]
 
     "## hello
 \t- world
@@ -244,7 +243,7 @@
 \t\t\t- nice
 \t\t\t- bingo
 \t\t\t- world"
-"## Halooooo
+    "## Halooooo
 \t- world
 \t\t- nice
 \t\t\t- nice
@@ -252,37 +251,37 @@
 \t\t\t- world"
 ;; Empty op, because no insertion op before the first base block required
 ;; See https://github.com/logseq/diff-merge#usage
-[[]
- [[-1 {:body "## hello"
-       :level 1
-       :uuid nil}]
-  [1  {:body "## Halooooo"
-       :level 1
-       :uuid nil}]
-  [1 {:body "world"
-      :level 2
-      :uuid nil}]]
- [[-1 {:body "world\nid:: 63e25526-3612-4fb1-8cf9-abcd12354abc"
-      :level 2
-      :uuid "63e25526-3612-4fb1-8cf9-abcd12354abc"}]]
- [[0 {:body "nice"
-      :level 3
-      :uuid nil}]]
- [[0 {:body "nice"
-      :level 4
-      :uuid nil}]]
- [[0 {:body "bingo"
-      :level 4
-      :uuid nil}]]
- [[0 {:body "world"
-      :level 4
-      :uuid nil}]]]
-
-""
-"- abc def"
-[[[1 {:body "abc def"
-      :level 1
-      :uuid nil}]]]))
+    [[]
+     [[-1 {:body "## hello"
+           :level 1
+           :uuid nil}]
+      [1  {:body "## Halooooo"
+           :level 1
+           :uuid nil}]
+      [1 {:body "world"
+          :level 2
+          :uuid nil}]]
+     [[-1 {:body "world\nid:: 63e25526-3612-4fb1-8cf9-abcd12354abc"
+           :level 2
+           :uuid "63e25526-3612-4fb1-8cf9-abcd12354abc"}]]
+     [[0 {:body "nice"
+          :level 3
+          :uuid nil}]]
+     [[0 {:body "nice"
+          :level 4
+          :uuid nil}]]
+     [[0 {:body "bingo"
+          :level 4
+          :uuid nil}]]
+     [[0 {:body "world"
+          :level 4
+          :uuid nil}]]]
+
+    ""
+    "- abc def"
+    [[[1 {:body "abc def"
+          :level 1
+          :uuid nil}]]]))
 
 (deftest db->diffblocks
   (let [conn (gp-db/start-conn)]
@@ -351,7 +350,7 @@
           diff-blocks)
     [[["Property_Drawer" [["foo" "#bar" [["Tag" [["Plain" "bar"]]]]] ["baz" "#bing" [["Tag" [["Plain" "bing"]]]]]]] {:start_pos 0, :end_pos 22}]]
     "foo:: #bar\nbaz:: #bing"
-     [{:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]))
+    [{:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]))
 
 (deftest ast-empty-diff-test
   (are [ast text diff-ops]
@@ -360,95 +359,7 @@
           diff-ops)
     [[["Property_Drawer" [["foo" "#bar" [["Tag" [["Plain" "bar"]]]]] ["baz" "#bing" [["Tag" [["Plain" "bing"]]]]]]] {:start_pos 0, :end_pos 22}]]
     "foo:: #bar\nbaz:: #bing"
-     [[[1 {:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]]]))
-
-;; Ensure diff-merge-uuids follows the id:: in the content
-(deftest diff-merge-uuid-extract-test
-  (let [conn (gp-db/start-conn)
-        foo-content (str "- abc
-  id:: 11451400-0000-0000-0000-000000000000\n"
-                 "- def
-  id:: 63246324-6324-6324-6324-632463246324\n")
-        bar-content (str "- ghi
-  id:: 11451411-1111-1111-1111-111111111111\n"
-                         "\t- jkl
-\t  id:: 63241234-1234-1234-1234-123412341234\n") ]
-    (graph-parser/parse-file conn "foo.md" foo-content {})
-    (graph-parser/parse-file conn "bar.md" bar-content {})
-    (are [ast content page-name uuids]
-         (= (with-redefs [conn/get-db (constantly @conn)]
-              (#'reset-file-handler/diff-merge-uuids-2ways :markdown ast content {:page-name page-name
-                                                                             :block-pattern "-"}))
-            uuids)
-
-      (gp-mldoc/->edn (str foo-content "- newline\n") (gp-mldoc/default-config :markdown))
-      (str foo-content "- newline\n")
-      "foo"
-      ["11451400-0000-0000-0000-000000000000"
-       "63246324-6324-6324-6324-632463246324"
-       nil]
-
-      (gp-mldoc/->edn (str bar-content "- newline\n") (gp-mldoc/default-config :markdown))
-      (str bar-content "- newline\n")
-      "bar"
-      ["11451411-1111-1111-1111-111111111111"
-       "63241234-1234-1234-1234-123412341234"
-       nil])))
-
-;; Ensure diff-merge-uuids keeps the block uuids unchanged at best effort
-(deftest diff-merge-uuid-persist-test
-  (let [conn (gp-db/start-conn)
-        foo-content (str "- abc\n"
-                         "- def\n")
-        bar-content (str "- ghi\n"
-                         "\t- jkl\n")
-        foo-new-content (str foo-content "- newline\n")
-        new-bar-content (str  "- newline\n" bar-content)]
-    (graph-parser/parse-file conn "foo-persist.md" foo-content {})
-    (graph-parser/parse-file conn "bar-persist.md" bar-content {})
-    ;; Compare if the uuids are the same as those inside DB when the modified content (adding new line) is parsed
-    (are [ast content page-name DB-uuids->new-uuids-fn]
-         (= (with-redefs [conn/get-db (constantly @conn)]
-              (#'reset-file-handler/diff-merge-uuids-2ways :markdown ast content {:page-name page-name
-                                                                             :block-pattern "-"}))
-            ;; Get all uuids under the page
-            (->> page-name
-                 (test-db->diff-blocks conn)
-                 (map :uuid)
-                 (vec)
-                 (DB-uuids->new-uuids-fn)))
-
-      ;; Append a new line to foo
-      (gp-mldoc/->edn foo-new-content (gp-mldoc/default-config :markdown))
-      foo-new-content
-      "foo-persist"
-      (fn [db-uuids] (conj db-uuids nil))
-
-      ;; Prepend a new line to bar
-      (gp-mldoc/->edn new-bar-content (gp-mldoc/default-config :markdown))
-      new-bar-content
-      "bar-persist"
-      (fn [db-uuids] (cons nil db-uuids)))))
-
-(deftest diff-merge-error-capture-test
-  ;; Any exception thrown in diff-merge-uuids-2ways should be captured and returned a nil
-  (let [conn (gp-db/start-conn)
-        foo-content (str "- abc\n"
-                         "- def\n")
-        foo-new-content (str foo-content "- newline\n")]
-    (graph-parser/parse-file conn "foo-error-cap.md" foo-content {})
-    (are [ast content page-name]
-         (= (with-redefs [conn/get-db (constantly @conn)
-                                ;; Hijack the function to throw an exception
-                          fs-diff/db->diff-blocks #(throw (js/Error. "intentional exception for testing diff-merge-uuids-2ways error capture"))]
-              (#'reset-file-handler/diff-merge-uuids-2ways :markdown ast content {:page-name page-name
-                                                                                   :block-pattern "-"}))
-            nil)
-
-            ;; Append a new line to foo
-      (gp-mldoc/->edn foo-new-content (gp-mldoc/default-config :markdown))
-      foo-new-content
-      "foo-error-cap")))
+    [[[1 {:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]]]))
 
 (deftest test-remove-indentation-spaces
   (is (= "" (gp-mldoc/remove-indentation-spaces "" 0 false)))