Merge branch 'feat/db' into feat/capacitor-new

.github/workflows/build.yml

@@ -2,6 +2,10 @@ name: CI
 
 on:
   push:
+    branches: [master]
+    paths-ignore:
+      - '*.md'
+  pull_request:
     branches: [master, "feat/db"]
     paths-ignore:
       - '*.md'

.github/workflows/clj-e2e.yml

@@ -2,6 +2,14 @@ name: Clojure E2E
 
 on:
   push:
+    branches: [master]
+    paths:
+      - 'clj-e2e/**'
+      - '.github/workflows/clj-e2e.yml'
+      - src/**
+      - deps/**
+      - packages/**
+  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'clj-e2e/**'

.github/workflows/clj-rtc-e2e.yml

@@ -2,6 +2,14 @@ name: Clojure RTC E2E
 
 on:
   push:
+    branches: [master]
+    paths:
+      - 'clj-e2e/**'
+      - '.github/workflows/clj-rtc-e2e.yml'
+      - src/**
+      - deps/**
+      - packages/**
+  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'clj-e2e/**'

.github/workflows/db.yml

@@ -3,6 +3,12 @@ name: logseq/db CI
 on:
   # Path filters ensure jobs only kick off if a change is made to db
   push:
+    branches: [master]
+    paths:
+      - 'deps/db/**'
+      - '.github/workflows/db.yml'
+      - '!deps/db/**.md'
+  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'deps/db/**'

.github/workflows/graph-parser.yml

@@ -4,13 +4,20 @@ on:
   # Path filters ensure jobs only kick off if a change is made to graph-parser or
   # its local dependencies
   push:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/graph-parser/**'
       # db is a local dep that could break functionality in this lib and should trigger this
       - 'deps/db/**'
       - '.github/workflows/graph-parser.yml'
       - '!deps/graph-parser/**.md'
+  pull_request:
+    branches: [master, "feat/db"]
+    paths:
+      - 'deps/graph-parser/**'
+      - 'deps/db/**'
+      - '.github/workflows/graph-parser.yml'
+      - '!deps/graph-parser/**.md'
 
 defaults:
   run:

.github/workflows/logseq-common.yml

@@ -3,6 +3,12 @@ name: logseq/common CI
 on:
   # Path filters ensure jobs only kick off if a change is made to common
   push:
+    branches: [master]
+    paths:
+      - 'deps/common/**'
+      - '.github/workflows/logseq-common.yml'
+      - '!deps/common/**.md'
+  pull_request:
     branches: [master, "feat/db"]
     paths:
       - 'deps/common/**'

.github/workflows/outliner.yml

@@ -4,13 +4,20 @@ on:
   # Path filters ensure jobs only kick off if a change is made to outliner or
   # its local dependencies
   push:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/outliner/**'
       # db is a local dep that could break functionality in this lib and should trigger this
       - 'deps/db/**'
       - '.github/workflows/outliner.yml'
       - '!deps/outliner/**.md'
+  pull_request:
+    branches: [master, "feat/db"]
+    paths:
+      - 'deps/outliner/**'
+      - 'deps/db/**'
+      - '.github/workflows/outliner.yml'
+      - '!deps/outliner/**.md'
 
 defaults:
   run:

.github/workflows/publishing.yml

@@ -4,13 +4,20 @@ on:
   # Path filters ensure jobs only kick off if a change is made to publishing or
   # its local dependencies
   push:
-    branches: [master, "feat/db"]
+    branches: [master]
     paths:
       - 'deps/publishing/**'
       # db is a local dep that could break functionality in this lib and should trigger this
       - 'deps/db/**'
       - '.github/workflows/publishing.yml'
       - '!deps/publishing/**.md'
+  pull_request:
+    branches: [master, "feat/db"]
+    paths:
+      - 'deps/publishing/**'
+      - 'deps/db/**'
+      - '.github/workflows/publishing.yml'
+      - '!deps/publishing/**.md'
 
 defaults:
   run:

clj-e2e/dev/user.clj

@@ -16,13 +16,14 @@
             [logseq.e2e.property-basic-test]
             [logseq.e2e.util :as util]
             [wally.main :as w]
-            [wally.repl :as repl]))
+            [wally.repl :as repl]
+            [logseq.e2e.locator :as loc]))
 
 ;; Use port 3001 for local testing
 (reset! config/*port 3001)
 ;; show ui
 (reset! config/*headless false)
-(reset! config/*slow-mo 50)
+(reset! config/*slow-mo 30)
 
 (def *futures (atom {}))
 

clj-e2e/src/logseq/e2e/block.clj

@@ -22,14 +22,18 @@
 
 (defn new-block
   [title]
-  (let [editor (util/get-editor)
-        blocks-count (util/blocks-count)]
+  (let [editor (util/get-editor)]
     (when-not editor (open-last-block))
     (assert/assert-editor-mode)
-    (k/enter)
-    (assert/assert-have-count ".ls-block" (inc blocks-count))
-    (assert/assert-editor-mode)
-    (save-block title)))
+    (let [last-id (.getAttribute (w/-query ".editor-wrapper textarea") "id")]
+      (is (some? last-id))
+      (k/enter)
+      (assert/assert-is-visible
+       (loc/filter ".editor-wrapper"
+                   :has "textarea"
+                   :has-not (str "#" last-id)))
+      (assert/assert-editor-mode)
+      (save-block title))))
 
 ;; TODO: support tree
 (defn new-blocks
@@ -38,7 +42,7 @@
     (when-not editor? (open-last-block))
     (assert/assert-editor-mode)
     (let [value (util/get-edit-content)]
-      (if (string/blank? value)           ; empty block
+      (if (string/blank? value)         ; empty block
         (save-block (first titles))
         (new-block (first titles))))
     (doseq [title (rest titles)]

clj-e2e/src/logseq/e2e/keyboard.clj

@@ -1,6 +1,8 @@
 (ns logseq.e2e.keyboard
   (:require [wally.main :as w]))
 
+(def ^:private mac? (= "Mac OS X" (System/getProperty "os.name")))
+
 (def press w/keyboard-press)
 
 (def enter #(press "Enter"))
@@ -9,5 +11,11 @@
 (def tab #(press "Tab"))
 (def shift+tab #(press "Shift+Tab"))
 (def shift+enter #(press "Shift+Enter"))
+(def shift+arrow-up #(press "Shift+ArrowUp"))
+(def shift+arrow-down #(press "Shift+ArrowDown"))
+
 (def arrow-up #(press "ArrowUp"))
 (def arrow-down #(press "ArrowDown"))
+
+(def meta+shift+arrow-up #(press (str (if mac? "Meta" "Alt") "+Shift+ArrowUp")))
+(def meta+shift+arrow-down #(press (str (if mac? "Meta" "Alt") "+Shift+ArrowDown")))

clj-e2e/test/logseq/e2e/rtc_extra_test.clj

@@ -7,6 +7,7 @@
    [logseq.e2e.custom-report :as custom-report]
    [logseq.e2e.fixtures :as fixtures :refer [*page1 *page2]]
    [logseq.e2e.graph :as graph]
+   [logseq.e2e.keyboard :as k]
    [logseq.e2e.locator :as loc]
    [logseq.e2e.outliner-basic-test :as outliner-basic-test]
    [logseq.e2e.page :as page]
@@ -215,24 +216,14 @@
                                (let [{:keys [_local-tx remote-tx]}
                                      (rtc/with-wait-tx-updated
                                        (test-fn))]
-                                 (reset! *latest-remote-tx remote-tx))))]
-
-      ;; testing while rtc connected
-      (let [*latest-remote-tx (atom nil)]
-        (new-logseq-page)
-        (test-fn-in-page2 *latest-remote-tx)
-        (w/with-page @*page1
-          (rtc/wait-tx-update-to @*latest-remote-tx))
-        (validate-2-graphs))
-
-      ;; testing while rtc off then on
-      (let [*latest-remote-tx (atom nil)]
-        (new-logseq-page)
-        (rtc/with-stop-restart-rtc
-          [@*page1]
-          [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
-          (test-fn-in-page2 *latest-remote-tx))
-        (validate-2-graphs)))))
+                                 (reset! *latest-remote-tx remote-tx))))
+          *latest-remote-tx (atom nil)]
+      (new-logseq-page)
+      (rtc/with-stop-restart-rtc
+        [@*page1]
+        [@*page1 (rtc/wait-tx-update-to @*latest-remote-tx)]
+        (test-fn-in-page2 *latest-remote-tx))
+      (validate-2-graphs))))
 
 (deftest rtc-outliner-conflict-update-test
   (let [title-prefix "rtc-outliner-conflict-update-test"]
@@ -246,15 +237,55 @@
         (w/with-page @*page2
           (rtc/wait-tx-update-to @*latest-remote-tx))
         (validate-2-graphs)))
-    (testing "disconnect on page1 and page2, do some conflict updates, reconnect and check"
+    (testing "page1: indent block1 as child of block0, page2: delete block0"
       (rtc/with-stop-restart-rtc
         [@*page1 @*page2]
-        [@*page1 (rtc/with-wait-tx-updated (b/new-block "xxxx"))
-         @*page2 (rtc/with-wait-tx-updated (b/new-block "yyyy"))]
+        [@*page1 (rtc/with-wait-tx-updated
+                   (k/esc)
+                   (assert/assert-in-normal-mode?)
+                   (b/new-block "page1-done-1"))
+         @*page2 (rtc/with-wait-tx-updated
+                   (k/esc)
+                   (assert/assert-in-normal-mode?)
+                   (b/new-block "page2-done-1"))]
         (w/with-page @*page1
           (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 1)))
           (b/indent))
         (w/with-page @*page2
           (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 0)))
           (b/delete-blocks)))
-      (validate-2-graphs))))
+      (validate-2-graphs))
+    (comment
+      "this case is failing now"
+      (testing "
+origin:
+- block2
+- block3
+- block4
+page1:
+- block2
+  - block3
+    - block4
+page2:
+;; block2 deleted
+- block4
+  - block3"
+        (rtc/with-stop-restart-rtc
+          [@*page1 @*page2]
+          [@*page1 (rtc/with-wait-tx-updated (b/new-block "page1-done-2"))
+           @*page2 (rtc/with-wait-tx-updated (b/new-block "page2-done-2"))]
+          (w/with-page @*page1
+            (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 3)))
+            (b/indent)
+            (k/arrow-down)
+            (b/indent)
+            (b/indent))
+          (w/with-page @*page2
+            (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 2)))
+            (b/delete-blocks)
+            (w/click (format ".ls-block :text('%s')" (str title-prefix "-" 3)))
+            (k/shift+arrow-down)
+            (k/meta+shift+arrow-down)
+            (k/enter)
+            (b/indent)))
+        (validate-2-graphs)))))

deps/common/src/logseq/common/util.cljs

@@ -292,13 +292,13 @@
       (:block/name page)))
 
 (defn string-join-path
-  #_:clj-kondo/ignore
   "Replace all `strings/join` used to construct paths with this function to reduce lint output.
   https://github.com/logseq/logseq/pull/8679"
   [parts]
+  #_:clj-kondo/ignore
   (string/join "/" parts))
 
-(def escape-chars "[]{}().+*?|$")
+(def ^:private escape-chars "\\[]{}().+*?|$^")
 
 (defn escape-regex-chars
   "Escapes characters in string `old-value"

deps/common/test/logseq/common/util_test.cljs

@@ -1,39 +1,47 @@
 (ns logseq.common.util-test
-  (:require [clojure.test :refer [deftest are]]
+  (:require [clojure.test :refer [deftest are testing]]
             [logseq.common.util :as common-util]))
 
 (deftest valid-edn-keyword?
   (are [x y]
        (= (common-util/valid-edn-keyword? x) y)
 
-       ":foo-bar"  true
-       ":foo!"     true
-       ":foo,bar"  false
-       "4"         false
-       "foo bar"   false
-       "`property" false))
+    ":foo-bar"  true
+    ":foo!"     true
+    ":foo,bar"  false
+    "4"         false
+    "foo bar"   false
+    "`property" false))
 
 (deftest extract-file-extension?
   (are [x y]
        (= (common-util/path->file-ext x) y)
-       "foo.bar" "bar"
-       "foo"     nil
-       "foo.bar.baz" "baz"
-       "../assets/audio.mp3" "mp3"
+    "foo.bar" "bar"
+    "foo"     nil
+    "foo.bar.baz" "baz"
+    "../assets/audio.mp3" "mp3"
        ;; From https://www.w3.org/TR/media-frags/
-       "../assets/audio.mp3?t=10,20" "mp3"
-       "../assets/audio.mp3?t=10,20#t=10" "mp3"
-       "/root/Documents/audio.mp3" "mp3"
-       "C:\\Users\\foo\\Documents\\audio.mp3" "mp3"
-       "/root/Documents/audio" nil
-       "/root/Documents/audio." nil
-       "special/characters/aäääöüß.7z" "7z"
-       "asldk lakls .lsad" "lsad"
-       "中文asldk lakls .lsad" "lsad"))
+    "../assets/audio.mp3?t=10,20" "mp3"
+    "../assets/audio.mp3?t=10,20#t=10" "mp3"
+    "/root/Documents/audio.mp3" "mp3"
+    "C:\\Users\\foo\\Documents\\audio.mp3" "mp3"
+    "/root/Documents/audio" nil
+    "/root/Documents/audio." nil
+    "special/characters/aäääöüß.7z" "7z"
+    "asldk lakls .lsad" "lsad"
+    "中文asldk lakls .lsad" "lsad"))
 
 (deftest url?
   (are [x y]
        (= (common-util/url? x) y)
-       "http://logseq.com" true
-       "prop:: value" false
-       "a:" false))
+    "http://logseq.com" true
+    "prop:: value" false
+    "a:" false))
+
+(deftest escape-regex-chars
+  (testing "ensure the result is a valid regex string"
+    (are [x]
+        (some? (re-find (re-pattern  (common-util/escape-regex-chars x)) x))
+      "[[page-name]]"
+      "end-with-backslash\\"
+      "\\[]{}().+*?|$^")))

deps/db/src/logseq/db/common/view.cljs

@@ -476,7 +476,10 @@
                                    [label value] (cond ref-type?
                                                        [(db-property/property-value-content e)
                                                         (select-keys e [:db/id :block/uuid])]
-                                                       (= :datetime (:logseq.property/type property))
+                                                       ;; FIXME: Move query concerns out of :label as UI labels are usually strings
+                                                       ;; All non-string values need to be passed to the query builder since non-ref prop values use the actual value
+                                                       ;; This check is less fragile than listing all the property types to support e.g. :datetime, :checkbox, :keyword, :any
+                                                       (not (string? v))
                                                        [v v]
                                                        :else
                                                        [(str v) v])]

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -51,7 +51,7 @@
   (let [existing-file-page (get-file-page db file-path)
         pages-to-clear (distinct (filter some? [existing-file-page (:db/id file-page)]))
         blocks (mapcat (fn [page-id]
-                         (ldb/get-page-blocks db page-id {:pull-keys [:db/id :block/uuid]}))
+                         (:block/_page (d/entity db page-id)))
                        pages-to-clear)
         retain-uuids (set (keep :block/uuid retain-uuid-blocks))]
     (retract-blocks-tx (distinct blocks) retain-uuids)))
@@ -63,12 +63,10 @@ Options available:
   * :delete-blocks-fn - Optional fn which is called with the new page, file and existing block uuids
   which may be referenced elsewhere. Used to delete the existing blocks before saving the new ones.
    Implemented in file-common-handler/validate-and-get-blocks-to-delete for IoC
-* :skip-db-transact? - Boolean which skips transacting in order to batch transactions. Default is false
-* :extract-options - Options map to pass to extract/extract"
+  * :extract-options - Options map to pass to extract/extract"
   ([conn file-path content] (parse-file conn file-path content {}))
-  ([conn file-path content {:keys [delete-blocks-fn extract-options skip-db-transact? ctime mtime]
-                            :or {delete-blocks-fn (constantly [])
-                                 skip-db-transact? false}
+  ([conn file-path content {:keys [delete-blocks-fn extract-options ctime mtime]
+                            :or {delete-blocks-fn (constantly [])}
                             :as options}]
    (let [format (common-util/get-format file-path)
          file-content [{:file/path file-path}]
@@ -91,7 +89,7 @@ Options available:
 
                      :else nil)
                block-ids (map (fn [block] {:block/uuid (:block/uuid block)}) blocks)
-               delete-blocks (delete-blocks-fn @conn (first pages) file-path block-ids)
+               delete-blocks (delete-blocks-fn (first pages) file-path block-ids)
                block-refs-ids (->> (mapcat :block/refs blocks)
                                    (filter (fn [ref] (and (vector? ref)
                                                           (= :block/uuid (first ref)))))
@@ -110,13 +108,9 @@ Options available:
                           (or ctime (nil? file-entity))
                           (assoc :file/created-at (or ctime (js/Date.)))
                           mtime
-                          (assoc :file/last-modified-at mtime))])
-         result (if skip-db-transact?
-                  tx
-                  (do
-                    (ldb/transact! conn tx (select-keys options [:new-graph? :from-disk?]))
-                    nil))]
-     {:tx result
+                          (assoc :file/last-modified-at mtime))])]
+     (ldb/transact! conn tx (select-keys options [:new-graph? :from-disk?]))
+     {:tx tx
       :ast ast})))
 
 (defn filter-files

deps/graph-parser/src/logseq/graph_parser/block.cljs

@@ -698,23 +698,15 @@
                                                    (str (gp-property/colons-org "id") " " (:block/uuid block)))))]
                                (string/replace-first c replace-str ""))))))
 
-(defn block-exists-in-another-page?
-  "For sanity check only.
-   For renaming file externally, the file is actually deleted and transacted before-hand."
-  [db block-uuid current-page-name]
-  (when (and db current-page-name)
-    (when-let [block-page-name (:block/name (:block/page (d/entity db [:block/uuid block-uuid])))]
-      (not= current-page-name block-page-name))))
-
 (defn fix-block-id-if-duplicated!
-  "If the block exists in another page, we need to fix it
-   If the block exists in the current extraction process, we also need to fix it"
-  [db page-name *block-exists-in-extraction block]
-  (let [block (if (or (@*block-exists-in-extraction (:block/uuid block))
-                      (block-exists-in-another-page? db (:block/uuid block) page-name))
+  "If the block exists in another page or the current page, we need to fix it"
+  [db page-name *extracted-block-ids block]
+  (let [block-page-name (:block/name (:block/page (d/entity db [:block/uuid (:block/uuid block)])))
+        block (if (or (and block-page-name (not= block-page-name page-name))
+                      (contains? @*extracted-block-ids (:block/uuid block)))
                 (fix-duplicate-id block)
                 block)]
-    (swap! *block-exists-in-extraction conj (:block/uuid block))
+    (swap! *extracted-block-ids conj (:block/uuid block))
     block))
 
 (defn extract-blocks

deps/graph-parser/src/logseq/graph_parser/cli.cljs

@@ -3,10 +3,10 @@
   (:require ["fs" :as fs]
             ["path" :as path]
             [clojure.edn :as edn]
-            [logseq.common.graph :as common-graph]
             [logseq.common.config :as common-config]
-            [logseq.graph-parser :as graph-parser]
+            [logseq.common.graph :as common-graph]
             [logseq.common.util :as common-util]
+            [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.db :as gp-db]))
 
 (defn- slurp
@@ -28,10 +28,10 @@
   [dir* config]
   (let [dir (path/resolve dir*)]
     (->> (common-graph/get-files dir)
-        (map #(hash-map :file/path %))
-        graph-parser/filter-files
-        (remove-hidden-files dir config)
-        (mapv #(assoc % :file/content (slurp (:file/path %)))))))
+         (map #(hash-map :file/path %))
+         graph-parser/filter-files
+         (remove-hidden-files dir config)
+         (mapv #(assoc % :file/content (slurp (:file/path %)))))))
 
 (defn- read-config
   "Reads repo-specific config from logseq/config.edn"
@@ -45,8 +45,7 @@
   [conn files {:keys [config] :as options}]
   (let [extract-options (merge {:date-formatter (common-config/get-date-formatter config)
                                 :user-config config
-                                :filename-format (or (:file/name-format config) :legacy)
-                                :extracted-block-ids (atom #{})}
+                                :filename-format (or (:file/name-format config) :legacy)}
                                (select-keys options [:verbose]))]
     (mapv
      (fn [{:file/keys [path content]}]

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -220,15 +220,13 @@
 (defn- extract-pages-and-blocks
   "uri-encoded? - if is true, apply URL decode on the file path
    options -
-     :extracted-block-ids - An atom that contains all block ids that have been extracted in the current page (not yet saved to db)
      :resolve-uuid-fn - Optional fn which is called to resolve uuids of each block. Enables diff-merge
        (2 ways diff) based uuid resolution upon external editing.
        returns a list of the uuids, given the receiving ast, or nil if not able to resolve.
        Implemented in reset-file-handler/diff-merge-uuids-2ways for IoC
        Called in gp-extract/extract as AST is being parsed and properties are extracted there"
-  [format ast properties file content {:keys [date-formatter db filename-format extracted-block-ids resolve-uuid-fn]
-                                       :or {extracted-block-ids (atom #{})
-                                            resolve-uuid-fn (constantly nil)}
+  [format ast properties file content {:keys [date-formatter db filename-format resolve-uuid-fn]
+                                       :or {resolve-uuid-fn (constantly nil)}
                                        :as options}]
   (assert db "Datascript DB is required")
   (try
@@ -237,9 +235,10 @@
           options' (assoc options :page-name page-name)
           ;; In case of diff-merge (2way) triggered, use the uuids to override the ones extracted from the AST
           override-uuids (resolve-uuid-fn format ast content options')
+          *extracted-block-ids (atom #{})
           blocks (->> (gp-block/extract-blocks ast content format options')
                       (attach-block-ids-if-match override-uuids)
-                      (mapv #(gp-block/fix-block-id-if-duplicated! db page-name extracted-block-ids %))
+                      (mapv #(gp-block/fix-block-id-if-duplicated! db page-name *extracted-block-ids %))
                       ;; FIXME: use page uuid
                       (gp-block/with-parent-and-order {:block/name page-name})
                       (vec))

deps/graph-parser/test/logseq/graph_parser_test.cljs

@@ -1,12 +1,12 @@
 (ns logseq.graph-parser-test
   (:require [cljs.test :refer [deftest testing is are]]
             [clojure.string :as string]
+            [datascript.core :as d]
+            [logseq.db :as ldb]
             [logseq.graph-parser :as graph-parser]
-            [logseq.graph-parser.db :as gp-db]
             [logseq.graph-parser.block :as gp-block]
-            [logseq.graph-parser.property :as gp-property]
-            [datascript.core :as d]
-            [logseq.db :as ldb]))
+            [logseq.graph-parser.db :as gp-db]
+            [logseq.graph-parser.property :as gp-property]))
 
 (def foo-edn
   "Example exported whiteboard page as an edn exportable."
@@ -78,7 +78,7 @@
                                                         (throw (js/Error "Testing unexpected failure")))]
         (try
           (parse-file conn "foo.md" "- id:: 628953c1-8d75-49fe-a648-f4c612109098"
-                      {:delete-blocks-fn (fn [_db page _file _uuids]
+                      {:delete-blocks-fn (fn [page _file _uuids]
                                            (reset! deleted-page page))})
           (catch :default _)))
       (is (= nil @deleted-page)

scripts/src/logseq/tasks/dev/lint.clj

@@ -62,12 +62,16 @@
   []
   (let [res (shell {:out :string :continue true}
                    "grep -r --exclude-dir=worker" "\\[frontend.worker.*:" "src/main/frontend")
-        req-lines (->> (:out res) string/split-lines)]
-    (if-not (and (= 1 (:exit res)) (= "" (:out res)))
-      (do
-        (println "The following worker requires should not be in frontend namespaces:")
-        (println (:out res))
-        (System/exit 1))
+        ;; allow reset-file b/c it's only affects tests
+        allowed-exceptions #{"src/main/frontend/handler/file_based/file.cljs:            [frontend.worker.file.reset :as file-reset]"}
+        invalid-lines (when (= 0 (:exit res))
+                        (remove #(some->> % (contains? allowed-exceptions))
+                                (string/split-lines (:out res))))
+        _ (when (> (:exit res) 1) (System/exit 1))]
+    (if (and (= 0 (:exit res)) (seq invalid-lines))
+      (do (println "The following worker requires should not be in frontend namespaces:")
+          (println (string/join "\n" invalid-lines))
+          (System/exit 1))
       (println "Valid frontend namespaces!"))))
 
 (defn worker-and-frontend-separate

src/main/frontend/components/query/builder.cljs

@@ -171,17 +171,20 @@
   [*property *private-property? *find *tree opts loc values {:keys [db-graph?]}]
   (let [values' (cons {:label "Select all"
                        :value "Select all"}
-                      values)
+                      (map #(hash-map :value (str (:value %))
+                                      ;; Preserve original-value as non-string values like boolean do not display in select
+                                      :original-value (:value %))
+                           values))
         find' (rum/react *find)]
     (select values'
-            (fn [{:keys [value]}]
+            (fn [{:keys [value original-value]}]
               (let [k (cond
                         db-graph? (if @*private-property? :private-property :property)
                         (= find' :page) :page-property
                         :else :property)
                     x (if (= value "Select all")
                         [k @*property]
-                        [k @*property value])]
+                        [k @*property original-value])]
                 (reset! *property nil)
                 (append-tree! *tree opts loc x))))))
 
@@ -193,7 +196,7 @@
      (fn [_property]
        (p/let [result (if db-graph?
                         (p/let [result (db-async/<get-property-values @*property)]
-                          (map (fn [{:keys [label _value]}]
+                          (map (fn [{:keys [label]}]
                                  {:label label
                                   :value label})
                                result))

src/main/frontend/handler/file_based/events.cljs

@@ -367,3 +367,15 @@
             nil
             template
             {:target page})))))))
+
+(defmethod events/handle :graph/backup-file [[_ repo file-path db-content]]
+  (p/let [disk-content (fs/read-file "" file-path)]
+    (fs/backup-db-file! repo file-path db-content disk-content)))
+
+(defmethod events/handle :graph/notify-existing-file [[_ data]]
+  (let [{:keys [current-file-path file-path]} data
+        error (t :file/validate-existing-file-error current-file-path file-path)]
+    (state/pub-event! [:notification/show
+                       {:content error
+                        :status :error
+                        :clear? false}])))

src/main/frontend/handler/file_based/file.cljs

@@ -7,7 +7,6 @@
             [frontend.db.file-based.model :as file-model]
             [frontend.fs :as fs]
             [frontend.handler.common.config-edn :as config-edn-common-handler]
-            [frontend.handler.file-based.reset-file :as reset-file-handler]
             [frontend.handler.global-config :as global-config-handler]
             [frontend.handler.repo-config :as repo-config-handler]
             [frontend.handler.ui :as ui-handler]
@@ -15,6 +14,7 @@
             [frontend.schema.handler.repo-config :as repo-config-schema]
             [frontend.state :as state]
             [frontend.util :as util]
+            [frontend.worker.file.reset :as file-reset]
             [lambdaisland.glogi :as log]
             [logseq.common.config :as common-config]
             [logseq.common.path :as path]
@@ -33,6 +33,12 @@
       (println "Load file failed: " path)
       (js/console.error e)))))
 
+(defn reset-file!
+  [repo file-path content opts]
+  (if util/node-test?
+    (file-reset/reset-file! repo (db/get-db repo false) file-path content opts)
+    (state/<invoke-db-worker :thread-api/reset-file repo file-path content opts)))
+
 (defn- load-multiple-files
   [repo-url paths]
   (doall
@@ -133,7 +139,7 @@
 (defn alter-file
   "Write any in-DB file, e.g. repo config, page, whiteboard, etc."
   [repo path content {:keys [reset? re-render-root? from-disk? skip-compare? new-graph? verbose
-                             skip-db-transact? extracted-block-ids ctime mtime]
+                             ctime mtime]
                       :fs/keys [event]
                       :or {reset? true
                            re-render-root? false
@@ -147,25 +153,22 @@
     (when (or config-valid? (not config-file?)) ; non-config file or valid config
       (let [opts {:new-graph? new-graph?
                   :from-disk? from-disk?
-                  :skip-db-transact? skip-db-transact?
                   :fs/event event
                   :ctime ctime
-                  :mtime mtime}
-            result (if reset?
-                     (do
-                       (when-not skip-db-transact?
-                         (when-let [page-id (file-model/get-file-page-id path)]
-                           (db/transact! repo
-                                         [[:db/retract page-id :block/alias]
-                                          [:db/retract page-id :block/tags]]
-                                         opts)))
-                       (reset-file-handler/reset-file!
-                        repo path content (merge opts
-                                                 ;; To avoid skipping the `:or` bounds for keyword destructuring
-                                                 (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids})
-                                                 (when (some? verbose) {:verbose verbose}))))
-                     (db/set-file-content! repo path content opts))]
-        (-> (p/let [_ (when-not from-disk?
+                  :mtime mtime}]
+        (-> (p/let [result (if reset?
+                             (p/do!
+                              (when-let [page-id (file-model/get-file-page-id path)]
+                                (db/transact! repo
+                                              [[:db/retract page-id :block/alias]
+                                               [:db/retract page-id :block/tags]]
+                                              opts))
+                              (reset-file!
+                               repo path content (merge opts
+                                                         ;; To avoid skipping the `:or` bounds for keyword destructuring
+                                                        (when (some? verbose) {:verbose verbose}))))
+                             (db/set-file-content! repo path content opts))
+                    _ (when-not from-disk?
                         (write-file-aux! repo path content {:skip-compare? skip-compare?}))]
               (when re-render-root? (ui-handler/re-render-root!))
 
@@ -178,14 +181,47 @@
 
                 (= path "logseq/config.edn")
                 (p/let [_ (repo-config-handler/restore-repo-config! repo content)]
-                  (state/pub-event! [:shortcut/refresh]))))
+                  (state/pub-event! [:shortcut/refresh])))
+
+              result)
             (p/catch
              (fn [error]
                (println "Write file failed, path: " path ", content: " content)
                (log/error :write/failed error)
                (state/pub-event! [:capture-error
                                   {:error error
-                                   :payload {:type :write-file/failed-for-alter-file}}]))))
+                                   :payload {:type :write-file/failed-for-alter-file}}]))))))))
+
+(defn alter-file-test-version
+  "Test version of alter-file that is synchronous"
+  [repo path content {:keys [reset? from-disk? new-graph? verbose
+                             ctime mtime]
+                      :fs/keys [event]
+                      :or {reset? true
+                           from-disk? false}}]
+  (let [path (common-util/path-normalize path)
+        config-file? (= path "logseq/config.edn")
+        _ (when config-file?
+            (detect-deprecations path content))
+        config-valid? (and config-file? (validate-file path content))]
+    (when (or config-valid? (not config-file?)) ; non-config file or valid config
+      (let [opts {:new-graph? new-graph?
+                  :from-disk? from-disk?
+                  :fs/event event
+                  :ctime ctime
+                  :mtime mtime}
+            result (if reset?
+                     (do
+                       (when-let [page-id (file-model/get-file-page-id path)]
+                         (db/transact! repo
+                                       [[:db/retract page-id :block/alias]
+                                        [:db/retract page-id :block/tags]]
+                                       opts))
+                       (reset-file!
+                        repo path content (merge opts
+                                                         ;; To avoid skipping the `:or` bounds for keyword destructuring
+                                                 (when (some? verbose) {:verbose verbose}))))
+                     (db/set-file-content! repo path content opts))]
         result))))
 
 (defn set-file-content!
@@ -234,10 +270,13 @@
                                 (map (fn [path] (db/get-file repo path)) paths)))]
     ;; update db
     (when update-db?
-      (doseq [[path content] files]
-        (if reset?
-          (reset-file-handler/reset-file! repo path content)
-          (db/set-file-content! repo path content))))
+      (p/all
+       (map
+        (fn [[path content]]
+          (if reset?
+            (reset-file! repo path content {})
+            (db/set-file-content! repo path content)))
+        files)))
     (alter-files-handler! repo files opts file->content)))
 
 (defn watch-for-current-graph-dir!

src/main/frontend/handler/file_based/repo.cljs

@@ -7,7 +7,6 @@
             [frontend.db.file-based.model :as file-model]
             [frontend.fs :as fs]
             [frontend.handler.file-based.file :as file-handler]
-            [frontend.handler.file-based.reset-file :as reset-file-handler]
             [frontend.handler.repo-config :as repo-config-handler]
             [frontend.handler.route :as route-handler]
             [frontend.handler.ui :as ui-handler]
@@ -38,7 +37,7 @@
         (p/let [_ (fs/mkdir-if-not-exists (path/path-join repo-dir pages-dir))
                 file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath default-content)]
           (when-not file-exists?
-            (reset-file-handler/reset-file! repo-url file-rpath default-content)))))))
+            (file-handler/reset-file! repo-url file-rpath default-content {})))))))
 
 (defn- create-custom-theme
   [repo-url]
@@ -50,7 +49,7 @@
     (p/let [_ (fs/mkdir-if-not-exists (path/path-join repo-dir config/app-name))
             file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath default-content)]
       (when-not file-exists?
-        (reset-file-handler/reset-file! repo-url path default-content)))))
+        (file-handler/reset-file! repo-url path default-content {})))))
 
 (comment
   (defn- create-dummy-notes-page
@@ -60,7 +59,7 @@
           file-rpath (str (config/get-pages-directory) "/how_to_make_dummy_notes.md")]
       (p/let [_ (fs/mkdir-if-not-exists (path/path-join repo-dir (config/get-pages-directory)))
               _file-exists? (fs/create-if-not-exists repo-url repo-dir file-rpath content)]
-        (reset-file-handler/reset-file! repo-url file-rpath content)))))
+        (file-handler/reset-file! repo-url file-rpath content {})))))
 
 (defn create-config-file-if-not-exists
   "Creates a default logseq/config.edn if it doesn't exist"
@@ -74,7 +73,7 @@
             path (str app-dir "/" config/config-file)]
         (p/let [file-exists? (fs/create-if-not-exists repo-url repo-dir "logseq/config.edn" default-content)]
           (when-not file-exists?
-            (reset-file-handler/reset-file! repo-url path default-content)
+            (file-handler/reset-file! repo-url path default-content {})
             (repo-config-handler/set-repo-config-state! repo-url default-content)))))))
 
 (defn- create-default-files!
@@ -89,36 +88,55 @@
            (create-custom-theme repo-url)
            (state/pub-event! [:page/create-today-journal repo-url]))))
 
-(defonce *file-tx (atom nil))
-
 (defn- parse-and-load-file!
   "Accept: .md, .org, .edn, .css"
-  [repo-url file {:keys [new-graph? verbose skip-db-transact? extracted-block-ids]
-                  :or {skip-db-transact? true}}]
+  [repo-url file {:keys [new-graph? verbose]}]
+  (->
+   (p/let [result (file-handler/alter-file repo-url
+                                           (:file/path file)
+                                           (:file/content file)
+                                           (merge (:stat file)
+                                                  {:new-graph? new-graph?
+                                                   :re-render-root? false
+                                                   :from-disk? true}
+                                                  ;; To avoid skipping the `:or` bounds for keyword destructuring
+                                                  (when (some? verbose) {:verbose verbose})))]
+     (state/set-parsing-state! (fn [m]
+                                 (update m :finished inc)))
+     result)
+   (p/catch (fn [e]
+              (println "Parse and load file failed: " (str (:file/path file)))
+              (js/console.error e)
+              (state/set-parsing-state! (fn [m]
+                                          (update m :failed-parsing-files conj [(:file/path file) e])))
+              (state/set-parsing-state! (fn [m]
+                                          (update m :finished inc)))
+              nil))))
+
+(defn- parse-and-load-file-test-version!
+  "Accept: .md, .org, .edn, .css"
+  [repo-url file {:keys [new-graph? verbose]}]
   (try
-    (reset! *file-tx
-            (file-handler/alter-file repo-url
-                                     (:file/path file)
-                                     (:file/content file)
-                                     (merge (:stat file)
-                                            {:new-graph? new-graph?
-                                             :re-render-root? false
-                                             :from-disk? true
-                                             :skip-db-transact? skip-db-transact?}
-                                            ;; To avoid skipping the `:or` bounds for keyword destructuring
-                                            (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids})
-                                            (when (some? verbose) {:verbose verbose}))))
-    (state/set-parsing-state! (fn [m]
-                                (update m :finished inc)))
-    @*file-tx
+    (let [result (file-handler/alter-file-test-version
+                  repo-url
+                  (:file/path file)
+                  (:file/content file)
+                  (merge (:stat file)
+                         {:new-graph? new-graph?
+                          :re-render-root? false
+                          :from-disk? true}
+                                                  ;; To avoid skipping the `:or` bounds for keyword destructuring
+                         (when (some? verbose) {:verbose verbose})))]
+      (state/set-parsing-state! (fn [m]
+                                  (update m :finished inc)))
+      result)
     (catch :default e
       (println "Parse and load file failed: " (str (:file/path file)))
       (js/console.error e)
       (state/set-parsing-state! (fn [m]
                                   (update m :failed-parsing-files conj [(:file/path file) e])))
       (state/set-parsing-state! (fn [m]
-                                  (update m :finished inc)))
-      nil)))
+                                  (update m :finished inc))))))
 
 (defn- after-parse
   [repo-url re-render? re-render-opts opts graph-added-chan]
@@ -145,8 +163,7 @@
         total (count supported-files)
         large-graph? (> total 1000)
         *page-names (atom #{})
-        *page-name->path (atom {})
-        *extracted-block-ids (atom #{})]
+        *page-name->path (atom {})]
     (when (seq delete-data) (db/transact! repo-url delete-data {:delete-files? true}))
     (state/set-current-repo! repo-url)
     (state/set-parsing-state! {:total (count supported-files)})
@@ -157,11 +174,9 @@
           (state/set-parsing-state! (fn [m]
                                       (assoc m
                                              :current-parsing-file (:file/path file))))
-          (parse-and-load-file! repo-url file (assoc
-                                               (select-keys opts [:new-graph? :verbose])
-                                               :skip-db-transact? false)))
+          (parse-and-load-file-test-version! repo-url file (select-keys opts [:new-graph? :verbose])))
         (after-parse repo-url re-render? re-render-opts opts graph-added-chan))
-      (async/go-loop [tx []]
+      (async/go-loop []
         (if-let [item (async/<! chan)]
           (let [[idx file] item
                 whiteboard? (common-config/whiteboard? (:file/path file))
@@ -174,13 +189,9 @@
 
             (when yield-for-ui? (async/<! (async/timeout 1)))
 
-            (let [opts' (-> (select-keys opts [:new-graph? :verbose])
-                            (assoc :extracted-block-ids *extracted-block-ids))
+            (let [opts' (select-keys opts [:new-graph? :verbose])
                   ;; whiteboards might have conflicting block IDs so that db transaction could be failed
-                  opts' (if whiteboard?
-                          (assoc opts' :skip-db-transact? false)
-                          opts')
-                  result (parse-and-load-file! repo-url file opts')
+                  result (async/<! (p->c (parse-and-load-file! repo-url file opts')))
                   page-name (when (coll? result) ; result could be a promise
                               (some (fn [x] (when (and (map? x)
                                                        (:block/title x)
@@ -188,30 +199,19 @@
                                               (:block/name x)))
                                     result))
                   page-exists? (and page-name (get @*page-names page-name))
-                  tx' (cond
-                        whiteboard? tx
-                        page-exists? (do
-                                       (state/pub-event! [:notification/show
-                                                          {:content [:div
-                                                                     (util/format "The file \"%s\" will be skipped because another file \"%s\" has the same page title."
-                                                                                  (:file/path file)
-                                                                                  (get @*page-name->path page-name))]
-                                                           :status :warning
-                                                           :clear? false}])
-                                       tx)
-                        :else (concat tx result))
+                  _ (when page-exists?
+                      (state/pub-event! [:notification/show
+                                         {:content [:div
+                                                    (util/format "The file \"%s\" will be skipped because another file \"%s\" has the same page title."
+                                                                 (:file/path file)
+                                                                 (get @*page-name->path page-name))]
+                                          :status :warning
+                                          :clear? false}]))
                   _ (when (and page-name (not page-exists?))
                       (swap! *page-names conj page-name)
-                      (swap! *page-name->path assoc page-name (:file/path file)))
-                  tx' (if (zero? (rem (inc idx) 100))
-                        (do
-                          (async/<! (p->c (db/transact! repo-url tx' {:from-disk? true})))
-                          [])
-                        tx')]
-              (recur tx')))
-          (p/do!
-           (when (seq tx) (db/transact! repo-url tx {:from-disk? true}))
-           (after-parse repo-url re-render? re-render-opts opts graph-added-chan)))))
+                      (swap! *page-name->path assoc page-name (:file/path file)))]
+              (recur)))
+          (after-parse repo-url re-render? re-render-opts opts graph-added-chan))))
     graph-added-chan))
 
 (defn- parse-files-and-create-default-files!

src/main/frontend/handler/file_based/reset_file.cljs

@@ -1,105 +0,0 @@
-(ns frontend.handler.file-based.reset-file
-  "Fns for resetting a db file with parsed file content"
-  (:require [frontend.config :as config]
-            [frontend.state :as state]
-            [frontend.db :as db]
-            [frontend.db.file-based.model :as file-model]
-            [logseq.graph-parser :as graph-parser]
-            [logseq.common.util :as common-util]
-            [frontend.fs.diff-merge :as diff-merge]
-            [frontend.fs :as fs]
-            [frontend.context.i18n :refer [t]]
-            [promesa.core :as p]
-            [clojure.string :as string]
-            [cljs-bean.core :as bean]
-            [lambdaisland.glogi :as log]))
-
-(defn- page-exists-in-another-file
-  "Conflict of files towards same page"
-  [repo-url page file]
-  (when-let [page-name (:block/name page)]
-    (let [current-file (:file/path (file-model/get-page-file repo-url page-name))]
-      (when (not= file current-file)
-        current-file))))
-
-(defn- validate-existing-file
-  "Handle the case when the file is already exists in db
-     Likely caused by renaming between caps and non-caps, then cause file system
-     bugs on some OS
-     e.g. on macOS, it doesn't fire the file change event when renaming between
-       caps and non-caps"
-  [repo-url file-page file-path]
-  (when-let [current-file (page-exists-in-another-file repo-url file-page file-path)]
-    (when (not= file-path current-file)
-      (cond
-        ;; TODO: handle case sensitive file system
-        (= (common-util/path-normalize (string/lower-case current-file))
-           (common-util/path-normalize (string/lower-case file-path)))
-        ;; case renamed
-        (when-let [file (db/pull [:file/path current-file])]
-          (p/let [disk-content (fs/read-file "" current-file)]
-            (fs/backup-db-file! repo-url current-file (:file/content file) disk-content))
-          (db/transact! repo-url [{:db/id (:db/id file)
-                                   :file/path file-path}]))
-
-        :else
-        (let [error (t :file/validate-existing-file-error current-file file-path)]
-          (state/pub-event! [:notification/show
-                             {:content error
-                              :status :error
-                              :clear? false}]))))))
-
-(defn- validate-and-get-blocks-to-delete
-  "An implementation for the delete-blocks-fn in graph-parser/parse-file"
-  [repo-url db file-page file-path retain-uuid-blocks]
-  (validate-existing-file repo-url file-page file-path)
-  (graph-parser/get-blocks-to-delete db file-page file-path retain-uuid-blocks))
-
-(defn- diff-merge-uuids-2ways
-  "Infer new uuids from existing DB data and diff with the new AST
-   Return a list of uuids for the new blocks"
-  [format ast content {:keys [page-name] :as options}]
-  (try
-    (let [base-diffblocks (diff-merge/db->diff-blocks page-name)
-          income-diffblocks (diff-merge/ast->diff-blocks ast content format options)
-          diff-ops (diff-merge/diff base-diffblocks income-diffblocks)
-          new-uuids (diff-merge/attachUUID diff-ops (map :uuid base-diffblocks))]
-      (bean/->clj new-uuids))
-    (catch js/Error e
-      (log/error :diff-merge/diff-merge-2way-calling-failed e))))
-
-(defn- reset-file!*
-  "Parse file considering diff-merge with local or remote file
-   Decide how to treat the parsed file based on the file's triggering event
-   options -
-     :fs/reset-event - the event that triggered the file update
-     :fs/local-file-change - file changed on local disk
-     :fs/remote-file-change - file changed on remote"
-  [repo-url file-path content {:fs/keys [event] :as options}]
-  (when-let [db-conn (db/get-db repo-url false)]
-    (case event
-      ;; the file is already in db, so we can use the existing file's blocks
-      ;; to do the diff-merge
-      :fs/local-file-change
-      (graph-parser/parse-file db-conn file-path content (assoc-in options [:extract-options :resolve-uuid-fn] diff-merge-uuids-2ways))
-
-      ;; default to parse the file
-      (graph-parser/parse-file db-conn file-path content options))))
-
-(defn reset-file!
-  "Main fn for updating a db with the results of a parsed file"
-  ([repo-url file-path content]
-   (reset-file! repo-url file-path content {}))
-  ([repo-url file-path content {:keys [verbose extracted-block-ids _ctime _mtime] :as options}]
-   (let [options (merge (dissoc options :verbose :extracted-block-ids)
-                        {:delete-blocks-fn (partial validate-and-get-blocks-to-delete repo-url)
-                         ;; Options here should also be present in gp-cli/parse-graph
-                         :extract-options (merge
-                                           {:user-config (state/get-config)
-                                            :date-formatter (state/get-date-formatter)
-                                            :block-pattern (config/get-block-pattern (common-util/get-format file-path))
-                                            :filename-format (state/get-filename-format repo-url)}
-                                           ;; To avoid skipping the `:or` bounds for keyword destructuring
-                                           (when (some? extracted-block-ids) {:extracted-block-ids extracted-block-ids})
-                                           (when (some? verbose) {:verbose verbose}))})]
-     (:tx (reset-file!* repo-url file-path content options)))))

src/main/frontend/handler/worker.cljs

@@ -49,6 +49,12 @@
 (defmethod handle :capture-error [_ _worker data]
   (state/pub-event! [:capture-error data]))
 
+(defmethod handle :backup-file [_ _worker data]
+  (state/pub-event! [:graph/backup-file data]))
+
+(defmethod handle :notify-existing-file  [_ _worker data]
+  (state/pub-event! [:graph/notify-existing-file data]))
+
 (defmethod handle :default [_ _worker data]
   (prn :debug "Worker data not handled: " data))
 

src/main/frontend/state.cljs

@@ -585,11 +585,6 @@ should be done through this fn in order to get global config and config defaults
     "LATER"
     "TODO"))
 
-(defn get-filename-format
-  ([] (get-filename-format (get-current-repo)))
-  ([repo]
-   (:file/name-format (get-config repo))))
-
 (defn get-date-formatter
   []
   (let [repo (get-current-repo)]

src/main/frontend/worker/db_worker.cljs

@@ -20,6 +20,7 @@
             [frontend.worker.db.validate :as worker-db-validate]
             [frontend.worker.export :as worker-export]
             [frontend.worker.file :as file]
+            [frontend.worker.file.reset :as file-reset]
             [frontend.worker.handler.page :as worker-page]
             [frontend.worker.handler.page.file-based.rename :as file-worker-page-rename]
             [frontend.worker.rtc.asset-db-listener]
@@ -190,44 +191,30 @@
                                                :db-schema-version (str version-in-db)}}))))
     missing-addresses))
 
-(def get-to-delete-unused-addresses-sql
-  "WITH to_delete(addr) AS (
-     SELECT value
-     FROM json_each(?)
-   ),
-  referenced(addr) AS (
-    SELECT json_each.value
-    FROM kvs
-    JOIN json_each(kvs.addresses)
-    WHERE kvs.addr NOT IN (SELECT addr FROM to_delete)
-      AND json_each.value IN (SELECT addr FROM to_delete)
-  )
-  SELECT addr FROM to_delete
-  WHERE addr NOT IN (SELECT addr FROM referenced)")
-
 (defn upsert-addr-content!
   "Upsert addr+data-seq. Update sqlite-cli/upsert-addr-content! when making changes"
   [db data delete-addrs*]
-  (let [delete-addrs (clojure.set/difference (set delete-addrs*) #{0 1})]
+  (let [_delete-addrs (clojure.set/difference (set delete-addrs*) #{0 1})]
     (assert (some? db) "sqlite db not exists")
     (.transaction db (fn [tx]
                        (doseq [item data]
                          (.exec tx #js {:sql "INSERT INTO kvs (addr, content, addresses) values ($addr, $content, $addresses) on conflict(addr) do update set content = $content, addresses = $addresses"
                                         :bind item}))))
-    (when (seq delete-addrs)
-      (let [result (.exec db #js {:sql get-to-delete-unused-addresses-sql
-                                  :bind (js/JSON.stringify (clj->js delete-addrs))
-                                  :rowMode "array"})
-            non-refed-addrs (map #(aget % 0) result)]
-        (when (seq non-refed-addrs)
-          (.transaction db (fn [tx]
-                             (doseq [addr non-refed-addrs]
-                               (.exec tx #js {:sql "Delete from kvs where addr = ?"
-                                              :bind #js [addr]}))))))
-      (let [missing-addrs (when worker-util/dev?
-                            (seq (find-missing-addresses nil db {:delete-addrs delete-addrs})))]
-        (when (seq missing-addrs)
-          (worker-util/post-message :notification [(str "Bug!! Missing addresses: " missing-addrs) :error false]))))))
+    ;; (when (seq delete-addrs)
+    ;;   (let [result (.exec db #js {:sql get-to-delete-unused-addresses-sql
+    ;;                               :bind (js/JSON.stringify (clj->js delete-addrs))
+    ;;                               :rowMode "array"})
+    ;;         non-refed-addrs (map #(aget % 0) result)]
+    ;;     (when (seq non-refed-addrs)
+    ;;       (.transaction db (fn [tx]
+    ;;                          (doseq [addr non-refed-addrs]
+    ;;                            (.exec tx #js {:sql "Delete from kvs where addr = ?"
+    ;;                                           :bind #js [addr]})))))
+    ;;     (let [missing-addrs (when worker-util/dev?
+    ;;                           (seq (find-missing-addresses nil db {:delete-addrs non-refed-addrs})))]
+    ;;       (when (seq missing-addrs)
+    ;;         (worker-util/post-message :notification [(str "Bug!! Missing addresses: " missing-addrs) :error false])))))
+    ))
 
 (defn restore-data-from-addr
   "Update sqlite-cli/restore-data-from-addr when making changes"
@@ -796,6 +783,12 @@
   [graph]
   (fix-broken-graph graph))
 
+(def-thread-api :thread-api/reset-file
+  [repo file-path content opts]
+  ;; (prn :debug :reset-file :file-path file-path :opts opts)
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (file-reset/reset-file! repo conn file-path content opts)))
+
 (comment
   (def-thread-api :general/dangerousRemoveAllDbs
     []
@@ -849,11 +842,11 @@
   [repo start-opts]
   (js/Promise.
    (m/sp
-    (c.m/<? (init-sqlite-module!))
-    (when-not (:import-type start-opts)
-      (c.m/<? (start-db! repo start-opts))
-      (assert (some? (worker-state/get-datascript-conn repo))))
-    (m/? (rtc.core/new-task--rtc-start true)))))
+     (c.m/<? (init-sqlite-module!))
+     (when-not (:import-type start-opts)
+       (c.m/<? (start-db! repo start-opts))
+       (assert (some? (worker-state/get-datascript-conn repo))))
+     (m/? (rtc.core/new-task--rtc-start true)))))
 
 (def broadcast-data-types
   (set (map

src/main/frontend/worker/file/reset.cljs

@@ -0,0 +1,79 @@
+(ns frontend.worker.file.reset
+  "Fns for resetting a db file with parsed file content"
+  (:require [clojure.string :as string]
+            [datascript.core :as d]
+            [frontend.worker.state :as worker-state]
+            [frontend.worker.util :as worker-util]
+            [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
+            [logseq.db :as ldb]
+            [logseq.graph-parser :as graph-parser]
+            [logseq.graph-parser.db :as gp-db]))
+
+(defn- page-exists-in-another-file
+  "Conflict of files towards same page"
+  [db page file]
+  (when-let [page-name (:block/name page)]
+    (let [current-file (:file/path (gp-db/get-page-file db page-name))]
+      (when (not= file current-file)
+        current-file))))
+
+(defn- validate-existing-file
+  "Handle the case when the file is already exists in db
+     Likely caused by renaming between caps and non-caps, then cause file system
+     bugs on some OS
+     e.g. on macOS, it doesn't fire the file change event when renaming between
+       caps and non-caps"
+  [repo conn file-page file-path]
+  (when-let [current-file-path (page-exists-in-another-file @conn file-page file-path)]
+    (when (not= file-path current-file-path)
+      (cond
+        ;; TODO: handle case sensitive file system
+        (= (common-util/path-normalize (string/lower-case current-file-path))
+           (common-util/path-normalize (string/lower-case file-path)))
+        ;; case renamed
+        (when-let [file (d/entity @conn [:file/path current-file-path])]
+          (worker-util/post-message :backup-file
+                                    [repo current-file-path (:file/content file)])
+          (ldb/transact! conn [{:db/id (:db/id file)
+                                :file/path file-path}]))
+
+        :else
+        (worker-util/post-message :notify-existing-file
+                                  [{:current-file-path current-file-path
+                                    :file-path file-path}])))))
+
+(defn- validate-and-get-blocks-to-delete
+  "An implementation for the delete-blocks-fn in graph-parser/parse-file"
+  [repo conn file-page file-path retain-uuid-blocks]
+  (validate-existing-file repo conn file-page file-path)
+  (graph-parser/get-blocks-to-delete @conn file-page file-path retain-uuid-blocks))
+
+(defn- reset-file!*
+  "Parse file.
+   Decide how to treat the parsed file based on the file's triggering event
+   options -
+     :fs/reset-event - the event that triggered the file update
+     :fs/local-file-change - file changed on local disk
+     :fs/remote-file-change - file changed on remote"
+  [db-conn file-path content options]
+  (graph-parser/parse-file db-conn file-path content options))
+
+(defn reset-file!
+  "Main fn for updating a db with the results of a parsed file"
+  ([repo conn file-path content]
+   (reset-file! repo conn file-path content {}))
+  ([repo conn file-path content {:keys [verbose _ctime _mtime] :as options}]
+   (let [config (worker-state/get-config repo)
+         options (merge (dissoc options :verbose)
+                        {:delete-blocks-fn (partial validate-and-get-blocks-to-delete repo conn)
+                         ;; Options here should also be present in gp-cli/parse-graph
+                         :extract-options (merge
+                                           {:user-config config
+                                            :date-formatter (worker-state/get-date-formatter repo)
+                                            :block-pattern (common-config/get-block-pattern
+                                                            (or (common-util/get-format file-path) :markdown))
+                                            :filename-format (:file/name-format config)}
+                                           ;; To avoid skipping the `:or` bounds for keyword destructuring
+                                           (when (some? verbose) {:verbose verbose}))})]
+     (:tx (reset-file!* conn file-path content options)))))

src/test/frontend/fs/diff_merge_test.cljs

@@ -3,7 +3,6 @@
             [cljs.test :refer [are deftest is]]
             [frontend.db.conn :as conn]
             [frontend.fs.diff-merge :as fs-diff]
-            [frontend.handler.file-based.reset-file :as reset-file-handler]
             [logseq.graph-parser :as graph-parser]
             [logseq.graph-parser.db :as gp-db]
             [logseq.graph-parser.mldoc :as gp-mldoc]
@@ -25,13 +24,13 @@
   (are [text diff-blocks]
        (= (org-text->diffblocks text)
           diff-blocks)
-        ":PROPERTIES:
+    ":PROPERTIES:
 :ID:       72289d9a-eb2f-427b-ad97-b605a4b8c59b
 :END:
 #+tItLe: Well parsed!"
-[{:body ":PROPERTIES:\n:ID:       72289d9a-eb2f-427b-ad97-b605a4b8c59b\n:END:\n#+tItLe: Well parsed!"
-  :uuid "72289d9a-eb2f-427b-ad97-b605a4b8c59b"
-  :level 1}]
+    [{:body ":PROPERTIES:\n:ID:       72289d9a-eb2f-427b-ad97-b605a4b8c59b\n:END:\n#+tItLe: Well parsed!"
+      :uuid "72289d9a-eb2f-427b-ad97-b605a4b8c59b"
+      :level 1}]
 
     "#+title: Howdy"
     [{:body "#+title: Howdy" :uuid nil :level 1}]
@@ -62,37 +61,37 @@
   (are [text diff-blocks]
        (= (text->diffblocks text)
           diff-blocks)
-  "- a
+    "- a
 \t- b
 \t\t- c"
-  [{:body "a" :uuid nil :level 1}
-   {:body "b" :uuid nil :level 2}
-   {:body "c" :uuid nil :level 3}]
+    [{:body "a" :uuid nil :level 1}
+     {:body "b" :uuid nil :level 2}
+     {:body "c" :uuid nil :level 3}]
 
-"- a
+    "- a
 \t- b
 \t\t- c
 \t\t  multiline
 - d"
-[{:body "a" :uuid nil :level 1}
- {:body "b" :uuid nil :level 2}
- {:body "c\nmultiline" :uuid nil :level 3}
- {:body "d" :uuid nil :level 1}]
+    [{:body "a" :uuid nil :level 1}
+     {:body "b" :uuid nil :level 2}
+     {:body "c\nmultiline" :uuid nil :level 3}
+     {:body "d" :uuid nil :level 1}]
 
-  "## hello
+    "## hello
 \t- world
 \t\t- nice
 \t\t\t- nice
 \t\t\t- bingo
 \t\t\t- world"
-  [{:body "## hello" :uuid nil :level 1}
-   {:body "world" :uuid nil :level 2}
-   {:body "nice" :uuid nil :level 3}
-   {:body "nice" :uuid nil :level 4}
-   {:body "bingo" :uuid nil :level 4}
-   {:body "world" :uuid nil :level 4}]
-
-  "# a
+    [{:body "## hello" :uuid nil :level 1}
+     {:body "world" :uuid nil :level 2}
+     {:body "nice" :uuid nil :level 3}
+     {:body "nice" :uuid nil :level 4}
+     {:body "bingo" :uuid nil :level 4}
+     {:body "world" :uuid nil :level 4}]
+
+    "# a
 ## b
 ### c
 #### d
@@ -102,27 +101,27 @@
 \t\t- h
 \t- i
 - j"
-  [{:body "# a" :uuid nil :level 1}
-   {:body "## b" :uuid nil :level 1}
-   {:body "### c" :uuid nil :level 1}
-   {:body "#### d" :uuid nil :level 1}
-   {:body "### e" :uuid nil :level 1}
-   {:body "f" :uuid nil :level 1}
-   {:body "g" :uuid nil :level 2}
-   {:body "h" :uuid nil :level 3}
-   {:body "i" :uuid nil :level 2}
-   {:body "j" :uuid nil :level 1}]
+    [{:body "# a" :uuid nil :level 1}
+     {:body "## b" :uuid nil :level 1}
+     {:body "### c" :uuid nil :level 1}
+     {:body "#### d" :uuid nil :level 1}
+     {:body "### e" :uuid nil :level 1}
+     {:body "f" :uuid nil :level 1}
+     {:body "g" :uuid nil :level 2}
+     {:body "h" :uuid nil :level 3}
+     {:body "i" :uuid nil :level 2}
+     {:body "j" :uuid nil :level 1}]
 
     "- a\n  id:: 63e25526-3612-4fb1-8cf9-f66db1254a58
 \t- b
 \t\t- c"
-[{:body "a\nid:: 63e25526-3612-4fb1-8cf9-f66db1254a58"
-  :uuid "63e25526-3612-4fb1-8cf9-f66db1254a58" :level 1}
- {:body "b" :uuid nil :level 2}
- {:body "c" :uuid nil :level 3}]
+    [{:body "a\nid:: 63e25526-3612-4fb1-8cf9-f66db1254a58"
+      :uuid "63e25526-3612-4fb1-8cf9-f66db1254a58" :level 1}
+     {:body "b" :uuid nil :level 2}
+     {:body "c" :uuid nil :level 3}]
 
-  "alias:: ⭐️\nicon:: ⭐️"
-[{:body "alias:: ⭐️\nicon:: ⭐️", :level 1, :uuid nil}]))
+    "alias:: ⭐️\nicon:: ⭐️"
+    [{:body "alias:: ⭐️\nicon:: ⭐️", :level 1, :uuid nil}]))
 
 (defn text->diffblocks-alt
   [text]
@@ -206,7 +205,7 @@
 \t\t\t- nice
 \t\t\t- bingo
 \t\t\t- world"
-      "## Halooooo
+    "## Halooooo
 \t- world
 \t\t- nice
 \t\t\t- nice
@@ -216,26 +215,26 @@
     ;; See https://github.com/logseq/diff-merge#usage
     [[]
      [[-1 {:body "## hello"
-          :level 1
-          :uuid nil}]
+           :level 1
+           :uuid nil}]
       [1  {:body "## Halooooo"
-          :level 1
-          :uuid nil}]]
+           :level 1
+           :uuid nil}]]
      [[0 {:body "world"
-         :level 2
-         :uuid nil}]]
+          :level 2
+          :uuid nil}]]
      [[0 {:body "nice"
-         :level 3
-         :uuid nil}]]
+          :level 3
+          :uuid nil}]]
      [[0 {:body "nice"
-         :level 4
-         :uuid nil}]]
+          :level 4
+          :uuid nil}]]
      [[0 {:body "bingo"
-         :level 4
-         :uuid nil}]]
+          :level 4
+          :uuid nil}]]
      [[0 {:body "world"
-         :level 4
-         :uuid nil}]]]
+          :level 4
+          :uuid nil}]]]
 
     "## hello
 \t- world
@@ -244,7 +243,7 @@
 \t\t\t- nice
 \t\t\t- bingo
 \t\t\t- world"
-"## Halooooo
+    "## Halooooo
 \t- world
 \t\t- nice
 \t\t\t- nice
@@ -252,37 +251,37 @@
 \t\t\t- world"
 ;; Empty op, because no insertion op before the first base block required
 ;; See https://github.com/logseq/diff-merge#usage
-[[]
- [[-1 {:body "## hello"
-       :level 1
-       :uuid nil}]
-  [1  {:body "## Halooooo"
-       :level 1
-       :uuid nil}]
-  [1 {:body "world"
-      :level 2
-      :uuid nil}]]
- [[-1 {:body "world\nid:: 63e25526-3612-4fb1-8cf9-abcd12354abc"
-      :level 2
-      :uuid "63e25526-3612-4fb1-8cf9-abcd12354abc"}]]
- [[0 {:body "nice"
-      :level 3
-      :uuid nil}]]
- [[0 {:body "nice"
-      :level 4
-      :uuid nil}]]
- [[0 {:body "bingo"
-      :level 4
-      :uuid nil}]]
- [[0 {:body "world"
-      :level 4
-      :uuid nil}]]]
-
-""
-"- abc def"
-[[[1 {:body "abc def"
-      :level 1
-      :uuid nil}]]]))
+    [[]
+     [[-1 {:body "## hello"
+           :level 1
+           :uuid nil}]
+      [1  {:body "## Halooooo"
+           :level 1
+           :uuid nil}]
+      [1 {:body "world"
+          :level 2
+          :uuid nil}]]
+     [[-1 {:body "world\nid:: 63e25526-3612-4fb1-8cf9-abcd12354abc"
+           :level 2
+           :uuid "63e25526-3612-4fb1-8cf9-abcd12354abc"}]]
+     [[0 {:body "nice"
+          :level 3
+          :uuid nil}]]
+     [[0 {:body "nice"
+          :level 4
+          :uuid nil}]]
+     [[0 {:body "bingo"
+          :level 4
+          :uuid nil}]]
+     [[0 {:body "world"
+          :level 4
+          :uuid nil}]]]
+
+    ""
+    "- abc def"
+    [[[1 {:body "abc def"
+          :level 1
+          :uuid nil}]]]))
 
 (deftest db->diffblocks
   (let [conn (gp-db/start-conn)]
@@ -351,7 +350,7 @@
           diff-blocks)
     [[["Property_Drawer" [["foo" "#bar" [["Tag" [["Plain" "bar"]]]]] ["baz" "#bing" [["Tag" [["Plain" "bing"]]]]]]] {:start_pos 0, :end_pos 22}]]
     "foo:: #bar\nbaz:: #bing"
-     [{:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]))
+    [{:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]))
 
 (deftest ast-empty-diff-test
   (are [ast text diff-ops]
@@ -360,95 +359,7 @@
           diff-ops)
     [[["Property_Drawer" [["foo" "#bar" [["Tag" [["Plain" "bar"]]]]] ["baz" "#bing" [["Tag" [["Plain" "bing"]]]]]]] {:start_pos 0, :end_pos 22}]]
     "foo:: #bar\nbaz:: #bing"
-     [[[1 {:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]]]))
-
-;; Ensure diff-merge-uuids follows the id:: in the content
-(deftest diff-merge-uuid-extract-test
-  (let [conn (gp-db/start-conn)
-        foo-content (str "- abc
-  id:: 11451400-0000-0000-0000-000000000000\n"
-                 "- def
-  id:: 63246324-6324-6324-6324-632463246324\n")
-        bar-content (str "- ghi
-  id:: 11451411-1111-1111-1111-111111111111\n"
-                         "\t- jkl
-\t  id:: 63241234-1234-1234-1234-123412341234\n") ]
-    (graph-parser/parse-file conn "foo.md" foo-content {})
-    (graph-parser/parse-file conn "bar.md" bar-content {})
-    (are [ast content page-name uuids]
-         (= (with-redefs [conn/get-db (constantly @conn)]
-              (#'reset-file-handler/diff-merge-uuids-2ways :markdown ast content {:page-name page-name
-                                                                             :block-pattern "-"}))
-            uuids)
-
-      (gp-mldoc/->edn (str foo-content "- newline\n") (gp-mldoc/default-config :markdown))
-      (str foo-content "- newline\n")
-      "foo"
-      ["11451400-0000-0000-0000-000000000000"
-       "63246324-6324-6324-6324-632463246324"
-       nil]
-
-      (gp-mldoc/->edn (str bar-content "- newline\n") (gp-mldoc/default-config :markdown))
-      (str bar-content "- newline\n")
-      "bar"
-      ["11451411-1111-1111-1111-111111111111"
-       "63241234-1234-1234-1234-123412341234"
-       nil])))
-
-;; Ensure diff-merge-uuids keeps the block uuids unchanged at best effort
-(deftest diff-merge-uuid-persist-test
-  (let [conn (gp-db/start-conn)
-        foo-content (str "- abc\n"
-                         "- def\n")
-        bar-content (str "- ghi\n"
-                         "\t- jkl\n")
-        foo-new-content (str foo-content "- newline\n")
-        new-bar-content (str  "- newline\n" bar-content)]
-    (graph-parser/parse-file conn "foo-persist.md" foo-content {})
-    (graph-parser/parse-file conn "bar-persist.md" bar-content {})
-    ;; Compare if the uuids are the same as those inside DB when the modified content (adding new line) is parsed
-    (are [ast content page-name DB-uuids->new-uuids-fn]
-         (= (with-redefs [conn/get-db (constantly @conn)]
-              (#'reset-file-handler/diff-merge-uuids-2ways :markdown ast content {:page-name page-name
-                                                                             :block-pattern "-"}))
-            ;; Get all uuids under the page
-            (->> page-name
-                 (test-db->diff-blocks conn)
-                 (map :uuid)
-                 (vec)
-                 (DB-uuids->new-uuids-fn)))
-
-      ;; Append a new line to foo
-      (gp-mldoc/->edn foo-new-content (gp-mldoc/default-config :markdown))
-      foo-new-content
-      "foo-persist"
-      (fn [db-uuids] (conj db-uuids nil))
-
-      ;; Prepend a new line to bar
-      (gp-mldoc/->edn new-bar-content (gp-mldoc/default-config :markdown))
-      new-bar-content
-      "bar-persist"
-      (fn [db-uuids] (cons nil db-uuids)))))
-
-(deftest diff-merge-error-capture-test
-  ;; Any exception thrown in diff-merge-uuids-2ways should be captured and returned a nil
-  (let [conn (gp-db/start-conn)
-        foo-content (str "- abc\n"
-                         "- def\n")
-        foo-new-content (str foo-content "- newline\n")]
-    (graph-parser/parse-file conn "foo-error-cap.md" foo-content {})
-    (are [ast content page-name]
-         (= (with-redefs [conn/get-db (constantly @conn)
-                                ;; Hijack the function to throw an exception
-                          fs-diff/db->diff-blocks #(throw (js/Error. "intentional exception for testing diff-merge-uuids-2ways error capture"))]
-              (#'reset-file-handler/diff-merge-uuids-2ways :markdown ast content {:page-name page-name
-                                                                                   :block-pattern "-"}))
-            nil)
-
-            ;; Append a new line to foo
-      (gp-mldoc/->edn foo-new-content (gp-mldoc/default-config :markdown))
-      foo-new-content
-      "foo-error-cap")))
+    [[[1 {:body "foo:: #bar\nbaz:: #bing", :level 1, :uuid nil}]]]))
 
 (deftest test-remove-indentation-spaces
   (is (= "" (gp-mldoc/remove-indentation-spaces "" 0 false)))

src/test/frontend/handler/repo_test.cljs

@@ -10,7 +10,8 @@
             [datascript.core :as d]
             [clojure.edn :as edn]
             ["path" :as node-path]
-            ["fs" :as fs]))
+            ["fs" :as fs]
+            [frontend.worker.state :as worker-state]))
 
 (use-fixtures :each test-helper/start-and-destroy-db)
 
@@ -19,7 +20,7 @@
         _ (docs-graph-helper/clone-docs-repo-if-not-exists graph-dir "v0.10.9")
         repo-config (edn/read-string (str (fs/readFileSync (node-path/join graph-dir "logseq/config.edn"))))
         files (#'gp-cli/build-graph-files graph-dir repo-config)
-        _ (test-helper/with-config repo-config
+        _ (with-redefs [worker-state/get-config (constantly repo-config)]
             (file-repo-handler/parse-files-and-load-to-db! test-helper/test-db files {:re-render? false :verbose false}))
         db (conn/get-db test-helper/test-db)]