enhance(sync): add merge step to update local

deps/graph-parser/src/logseq/graph_parser/util.cljs

@@ -213,6 +213,7 @@
   (second (re-find #"(?:\.)(\w+)[^.]*$" path-or-file-name)))
 
 (defn get-format
+  "File path to format keyword, :org, :markdown, etc."
   [file]
   (when file
     (normalize-format (keyword (some-> (path->file-ext file) string/lower-case)))))

src/electron/electron/file_sync_rsapi.cljs

@@ -26,6 +26,9 @@
 (defn delete-local-files [graph-uuid base-path file-paths]
   (rsapi/deleteLocalFiles graph-uuid base-path (clj->js file-paths)))
 
+(defn fetch-remote-files [graph-uuid base-path file-paths token]
+  (rsapi/fetchRemoteFiles graph-uuid base-path (clj->js file-paths) token))
+
 (defn update-local-files [graph-uuid base-path file-paths token]
   (rsapi/updateLocalFiles graph-uuid base-path (clj->js file-paths) token))
 

src/electron/electron/handler.cljs

@@ -96,6 +96,10 @@
     (catch :default _e
       false)))
 
+(defmethod handle :copyFile [_window [_ _repo from-path to-path]]
+  (logger/info ::copy-file from-path to-path)
+  (fs-extra/copy from-path to-path))
+
 (defmethod handle :writeFile [window [_ repo path content]]
   (let [^js Buf (.-Buffer buffer)
         ^js content (if (instance? js/ArrayBuffer content)
@@ -636,6 +640,9 @@
 (defmethod handle :delete-local-files [_ args]
   (apply rsapi/delete-local-files (rest args)))
 
+(defmethod handle :fetch-remote-files [_ args]
+  (apply rsapi/fetch-remote-files (rest args)))
+
 (defmethod handle :update-local-files [_ args]
   (apply rsapi/update-local-files (rest args)))
 

src/main/frontend/fs/diff_merge.cljs

@@ -1,24 +1,17 @@
 (ns frontend.fs.diff-merge
   "Implementation of text (file) based content diff & merge for conflict resolution"
-  (:require ["@logseq/diff-merge" :refer [Differ attach_uuids]]
+  (:require ["@logseq/diff-merge" :refer [attach_uuids Differ Merger]]
+            [cljs-bean.core :as bean]
+            [frontend.db.model :as db-model]
+            [frontend.db.utils :as db-utils]
             [logseq.graph-parser.block :as gp-block]
+            [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.utf8 :as utf8]
-            [cljs-bean.core :as bean]
-            [frontend.db.utils :as db-utils]
-            [frontend.db.model :as db-model]))
+            [clojure.string :as string]))
 
-;; (defn diff-merge
-;;   "N-ways diff & merge
-;;    Accept: blocks
-;;    https://github.com/logseq/diff-merge/blob/44546f2427f20bd417b898c8ba7b7d10a9254774/lib/mldoc.ts#L17-L22
-;;    https://github.com/logseq/diff-merge/blob/85ca7e9bf7740d3880ed97d535a4f782a963395d/lib/merge.ts#L40"
-;;   [base & branches]
-;;   ()
-;;   (let [merger (Merger.)]
-;;     (.mergeBlocks merger (bean/->js base) (bean/->js branches))))
 
-(defn diff 
+(defn diff
   "2-ways diff
    Accept: blocks in the struct with the required info
    Please refer to the `Block` struct in the link below
@@ -95,3 +88,50 @@
                    :level 1
                    :uuid uuid}
                   (reverse headings))))))))
+
+
+(defn- rebuild-content
+  "translate [[[op block]]] to merged content"
+  [_base-diffblocks diffs format]
+  ;; [[[0 {:body "attrib:: xxx", :level 1, :uuid nil}] ...] ...]
+  (let  [level-prefix-fn (fn [level]
+                           (when (and (= format :markdown) (not= level 1))
+                             (apply str (repeat (dec level) "\t"))))
+         ops-fn (fn [ops]
+                  (map (fn [[op {:keys [body level]}]]
+                         (when (or (= op 0) (= op 1)) ;; equal or insert
+                           (str (level-prefix-fn level) body)))
+                       ops))]
+    (->> diffs
+         (mapcat ops-fn)
+         (filter seq)
+         (string/join "\n"))))
+
+
+(defn three-way-merge
+  [base income current format]
+  (let [->ast (fn [text] (if (= format :org)
+                           (gp-mldoc/->edn text (gp-mldoc/default-config :org))
+                           (gp-mldoc/->edn text (gp-mldoc/default-config :markdown))))
+        merger (Merger.)
+        base-ast (->ast base)
+        base-diffblocks (ast->diff-blocks base-ast base format {})
+        income-ast (->ast income)
+        income-diffblocks (ast->diff-blocks income-ast income format {})
+        current-ast (->ast current)
+        current-diffblocks (ast->diff-blocks current-ast current format {})
+        branch-diffblocks [income-diffblocks current-diffblocks]
+        merged (.mergeBlocks merger (bean/->js base-diffblocks) (bean/->js branch-diffblocks))
+        merged-diff (bean/->clj merged)
+        merged-content (rebuild-content base-diffblocks merged-diff format)]
+    merged-content))
+
+;; (defn diff-merge
+;;   "N-ways diff & merge
+;;    Accept: blocks
+;;    https://github.com/logseq/blob/44546f2427f20bd417b898c8ba7b7d10a9254774/lib/mldoc.ts#L17-L22
+;;    https://github.com/logseq/blob/85ca7e9bf7740d3880ed97d535a4f782a963395d/lib/merge.ts#L40"
+;;   [base & branches]
+;;   ()
+;;   (let [merger (Merger.)]
+;;     (.mergeBlocks merger (bean/->js base) (bean/->js branches))))

src/main/frontend/fs/node.cljs

@@ -118,7 +118,10 @@
 
   (rename! [_this _repo old-path new-path]
     (ipc/ipc "rename" old-path new-path))
-
+  ;; copy with overwrite, without confirmation
+  (copy! [_this repo old-path new-path]
+         (prn ::copy-file old-path new-path)
+    (ipc/ipc "copyFile" repo old-path new-path))
   (stat [_this fpath]
     (-> (ipc/ipc "stat" fpath)
         (p/then bean/->clj)))

src/main/frontend/fs/sync.cljs

@@ -1,41 +1,45 @@
 (ns frontend.fs.sync
   "Main ns for providing file sync functionality"
-  (:require [cljs-http.client :as http]
+  (:require ["@capawesome/capacitor-background-task" :refer [BackgroundTask]]
+            ["path" :as node-path]
+            [cljs-http.client :as http]
+            [cljs-time.coerce :as tc]
             [cljs-time.core :as t]
             [cljs-time.format :as tf]
-            [cljs-time.coerce :as tc]
-            [cljs.core.async :as async :refer [go timeout go-loop offer! poll! chan <! >!]]
+            [cljs.core.async :as async :refer [<! >! chan go go-loop offer!
+                                               poll! timeout]]
             [cljs.core.async.impl.channels]
             [cljs.core.async.interop :refer [p->c]]
             [cljs.spec.alpha :as s]
+            [clojure.pprint :as pp]
             [clojure.set :as set]
             [clojure.string :as string]
-            [clojure.pprint :as pp]
             [electron.ipc :as ipc]
-            [goog.string :as gstring]
             [frontend.config :as config]
+            [frontend.context.i18n :refer [t]]
+            [frontend.db :as db]
             [frontend.debug :as debug]
+            [frontend.diff :as diff]
+            [frontend.encrypt :as encrypt]
+            [frontend.fs :as fs]
+            [frontend.fs.capacitor-fs :as capacitor-fs]
+            [frontend.fs.diff-merge :as diff-merge]
+            [frontend.handler.file :as file-handler]
+            [frontend.handler.notification :as notification]
             [frontend.handler.user :as user]
-            [frontend.state :as state]
             [frontend.mobile.util :as mobile-util]
+            [frontend.pubsub :as pubsub]
+            [frontend.state :as state]
             [frontend.util :as util]
-            [frontend.util.persist-var :as persist-var]
             [frontend.util.fs :as fs-util]
-            [frontend.handler.notification :as notification]
-            [frontend.context.i18n :refer [t]]
-            [frontend.diff :as diff]
-            [frontend.db :as db]
-            [frontend.fs :as fs]
-            [frontend.encrypt :as encrypt]
-            [frontend.pubsub :as pubsub]
+            [frontend.util.persist-var :as persist-var]
+            [goog.string :as gstring]
+            [lambdaisland.glogi :as log]
+            [logseq.common.path :as path]
             [logseq.graph-parser.util :as gp-util]
             [medley.core :refer [dedupe-by]]
-            [rum.core :as rum]
             [promesa.core :as p]
-            [lambdaisland.glogi :as log]
-            [frontend.fs.capacitor-fs :as capacitor-fs]
-            ["@capawesome/capacitor-background-task" :refer [BackgroundTask]]
-            ["path" :as node-path]))
+            [rum.core :as rum]))
 
 ;;; ### Commentary
 ;; file-sync related local files/dirs:
@@ -719,6 +723,7 @@
   (<get-local-all-files-meta [this graph-uuid base-path] "get all local files' metadata")
   (<rename-local-file [this graph-uuid base-path from to])
   (<update-local-files [this graph-uuid base-path filepaths] "remote -> local")
+  (<fetch-remote-files [this graph-uuid base-path filepaths] "remote -> local version-db")
   (<download-version-files [this graph-uuid base-path filepaths])
   (<delete-local-files [this graph-uuid base-path filepaths])
   (<update-remote-files [this graph-uuid base-path filepaths local-txid] "local -> remote, return err or txid")
@@ -850,6 +855,13 @@
       (<! (<rsapi-cancel-all-requests))
       (let [token (<! (<get-token this))]
         (<! (p->c (ipc/ipc "update-local-files" graph-uuid base-path filepaths token))))))
+  (<fetch-remote-files [this graph-uuid base-path filepaths]
+    (println "fetch-remote-files" graph-uuid base-path filepaths)
+    (go
+      (<! (<rsapi-cancel-all-requests))
+      (let [token (<! (<get-token this))]
+        (<! (p->c (ipc/ipc "fetch-remote-files" graph-uuid base-path filepaths token))))))
+
   (<download-version-files [this graph-uuid base-path filepaths]
     (go
       (let [token (<! (<get-token this))
@@ -944,7 +956,9 @@
                                                                      :basePath base-path
                                                                      :filePaths filepaths'
                                                                      :token token})))))))
-
+  (<fetch-remote-files [this graph-uuid base-path filepaths]
+    (js/console.error "unimpl")
+    (prn ::todo))
   (<download-version-files [this graph-uuid base-path filepaths]
     (go
       (let [token (<! (<get-token this))
@@ -1539,6 +1553,63 @@
                            delete-filetxns)]
     (set (concat update-file-items rename-file-items delete-file-items))))
 
+
+(defn- <fetch-remote-and-update-local-files
+  [graph-uuid base-path relative-paths]
+  (go
+    (let [fetched-file-rpaths (<! (<fetch-remote-files rsapi graph-uuid base-path relative-paths))]
+      (p->c (p/all (->> fetched-file-rpaths
+                        (map (fn [rpath]
+                               (p/let [incoming-file (path/path-join "logseq/version-files/incoming" rpath)
+                                       base-file (path/path-join "logseq/version-files/base" rpath)
+                                       current-change-file rpath
+                                       format (gp-util/get-format current-change-file)
+                                       repo (state/get-current-repo)
+                                       repo-dir (config/get-repo-dir repo)
+                                       base-exists? (fs/file-exists? repo-dir base-file)]
+                                 (cond
+                                   base-exists?
+                                   (p/let [base-content (fs/read-file repo-dir base-file)
+                                           current-content (fs/read-file repo-dir current-change-file)]
+                                     (if (= base-content current-content)
+                                       (do
+                                         (prn "base=current, write directly")
+                                         (p/do!
+                                          (fs/copy! repo
+                                                    (path/path-join repo-dir incoming-file)
+                                                    (path/path-join repo-dir current-change-file))
+                                          (fs/copy! repo
+                                                   (path/path-join repo-dir incoming-file)
+                                                   (path/path-join repo-dir base-file))))
+                                       (do
+                                         (prn "base!=current, should do a 3-way merge")
+                                         (p/let [incoming-content (fs/read-file repo-dir incoming-file)
+                                                 merged-content (diff-merge/three-way-merge base-content incoming-content current-content format)]
+                                           (prn ::merged-content merged-content)
+                                           (when (seq merged-content)
+                                             (p/do!
+                                              (fs/write-file! repo repo-dir current-change-file merged-content {:skip-compare? true})
+                                              (file-handler/alter-file repo current-change-file merged-content {:re-render-root? true
+                                                                                                                :from-disk? true
+                                                                                                                :fs/event :fs/remote-file-change})))
+                                           ;; now, let fs watcher handle the rest uploading
+                                           (comment fs/copy! repo-url
+                                                    (path/path-join repo-dir incoming-file)
+                                                    (path/path-join repo-dir current-change-file))))))
+
+                                   :else
+                                   (do
+                                     (prn "no base, use legacy buggy mode")
+                                     (prn ::copy incoming-file current-change-file)
+                                     (fs/copy! repo
+                                               (path/path-join repo-dir incoming-file)
+                                               (path/path-join repo-dir current-change-file))
+                                     (fs/copy! repo
+                                               (path/path-join repo-dir incoming-file)
+                                               (path/path-join repo-dir base-file)))))))))))))
+  
+
+
 (defn- apply-filetxns
   [*sync-state graph-uuid base-path filetxns *paused]
   (go
@@ -1574,7 +1645,7 @@
                 (swap! *sync-state sync-state--remove-recent-remote->local-files
                        [recent-remote->local-file-item])))))
 
-        (let [update-local-files-ch (<update-local-files rsapi graph-uuid base-path (map relative-path filetxns))
+        (let [update-local-files-ch (<fetch-remote-and-update-local-files graph-uuid base-path (map relative-path filetxns))
               r (<! (<with-pause update-local-files-ch *paused))]
           (doseq [[filetxn origin-db-content] txn->db-content-vec]
             (when (<! (need-add-version-file? filetxn origin-db-content))

src/main/frontend/handler/common/file.cljs

@@ -72,8 +72,8 @@
    Decide how to treat the parsed file based on the file's triggering event
    options - 
      :fs/reset-event - the event that triggered the file update
-       :fs/local-file-change - file changed on local disk
-       :fs/remote-file-change - file changed on remote"
+     :fs/local-file-change - file changed on local disk
+     :fs/remote-file-change - file changed on remote"
   [repo-url file-path content {:fs/keys [event] :as options}]
   (let [db-conn (db/get-db repo-url false)]
     (case event