enhance(sync): create version files when sync conflict

src/electron/electron/handler.cljs

@@ -82,24 +82,36 @@
     (path/join (.-dir parsed-path)
                (.-name parsed-path))))
 
-(defn backup-file
-  [repo path content]
-  (let [path-dir (get-backup-dir repo path)
-        ext (path/extname path)
-        new-path (path/join path-dir
-                            (str (string/replace (.toISOString (js/Date.)) ":" "_")
-                                 ext))]
-    (fs-extra/ensureDirSync path-dir)
+(defn- get-version-file-dir
+  [repo path]
+  (let [path (string/replace path repo "")
+        dir (path/join repo "version-files/local")
+        path (path/join dir path)
+        parsed-path (path/parse path)]
+    (path/join (.-dir parsed-path)
+               (.-name parsed-path))))
+
+(defn- backup-file
+  [path ext content]
+  (let [new-path (path/join
+                  path
+                  (str (string/replace (.toISOString (js/Date.)) ":" "_")
+                       ext))]
+    (fs-extra/ensureDirSync path)
     (fs/writeFileSync new-path content)
     (fs/statSync new-path)
-    (truncate-old-versioned-files! path-dir)
+    (truncate-old-versioned-files! path)
     new-path))
 
 (defmethod handle :backupDbFile [_window [_ repo path db-content new-content]]
   (when (and (string? db-content)
              (string? new-content)
              (string-some-deleted? db-content new-content))
-    (backup-file repo path db-content)))
+    (backup-file (get-backup-dir repo path) (path/extname path) db-content)))
+
+
+(defmethod handle :addVersionFile [_window [_ repo path content]]
+  (backup-file (get-version-file-dir repo path) (path/extname path) content))
 
 (defmethod handle :openFileBackupDir [_window [_ repo path]]
   (when (string? path)
@@ -129,7 +141,7 @@
       (fs/statSync path)
       (catch :default e
         (let [backup-path (try
-                            (backup-file repo path content)
+                            (backup-file (get-backup-dir repo path) (path/extname path) content)
                             (catch :default e
                               (println "Backup file failed")
                               (js/console.dir e)))]

src/main/frontend/diff.cljs

@@ -10,7 +10,7 @@
 
 (defn diff
   [s1 s2]
-  (-> ((gobj/get jsdiff "diffLines") s1 s2)
+  (-> ((gobj/get jsdiff "diffLines") s1 s2 (clj->js {"newlineIsToken" true}))
       bean/->clj))
 
 (def inline-special-chars

src/main/frontend/fs/sync.cljs

@@ -18,6 +18,9 @@
             [frontend.util.persist-var :as persist-var]
             [frontend.handler.notification :as notification]
             [frontend.context.i18n :refer [t]]
+            [frontend.diff :as diff]
+            [frontend.db :as db]
+            [frontend.fs :as fs]
             [medley.core :refer [dedupe-by]]
             [rum.core :as rum]))
 
@@ -162,7 +165,7 @@
                                          (offer! remote-changes-chan data)))))))
 
 (defn ws-listen!
-  "return channal which output messages from server"
+  "return channel which output messages from server"
   [graph-uuid *ws]
   (let [remote-changes-chan (chan (async/sliding-buffer 1))]
     (ws-listen!* graph-uuid *ws remote-changes-chan)
@@ -721,23 +724,67 @@
 
 (def remoteapi (->RemoteAPI))
 
+(defn- add-new-version-file
+  [repo path content]
+  (go
+    (println "add-new-version-file: "
+             (<! (p->c (ipc/ipc "addVersionFile" (config/get-local-dir repo) path content))))))
+
+(defn- is-journals-or-pages?
+  [filetxn]
+  (let [rel-path (relative-path filetxn)]
+    (or (string/starts-with? rel-path "journals/")
+        (string/starts-with? rel-path "pages/"))))
+
+(defn- need-add-version-file?
+  "when we need to create a new version file:
+  1. when apply a 'update' filetxn, it already exists(same page name) locally and has delete diffs
+  2. when apply a 'delete' filetxn, its origin remote content and local content are different
+     - TODO: we need to store origin remote content md5 in server db
+  3. create version files only for files under 'journals/', 'pages/' dir"
+  [^FileTxn filetxn origin-db-content]
+  (go
+    (cond
+      (.renamed? filetxn)
+      false
+      (.-deleted? filetxn)
+      false
+      (.-updated? filetxn)
+      (let [path (relative-path filetxn)
+            repo (state/get-current-repo)
+            file-path (config/get-file-path repo path)
+            content (<! (p->c (fs/read-file "" file-path)))]
+        (or (nil? content)
+            (some :removed (diff/diff origin-db-content content)))))))
+
 (defn- apply-filetxns
   [graph-uuid base-path filetxns]
-  (cond
-    (.renamed? (first filetxns))
-    (let [filetxn (first filetxns)]
-      (assert (= 1 (count filetxns)))
-      (rename-local-file rsapi graph-uuid base-path
-                         (relative-path (.-from-path filetxn))
-                         (relative-path (.-to-path filetxn))))
-
-    (.-updated? (first filetxns))
-    (update-local-files rsapi graph-uuid base-path (map relative-path filetxns))
-
-    (.-deleted? (first filetxns))
-    (let [filetxn (first filetxns)]
-      (assert (= 1 (count filetxns)))
-      (go
+  (go
+    (cond
+      (.renamed? (first filetxns))
+      (let [^FileTxn filetxn (first filetxns)
+            from-path (.-from-path filetxn)
+            to-path (.-to-path filetxn)]
+        (assert (= 1 (count filetxns)))
+        (<! (rename-local-file rsapi graph-uuid base-path
+                               (relative-path from-path)
+                               (relative-path to-path))))
+
+      (.-updated? (first filetxns))
+      (let [repo (state/get-current-repo)
+            txn->db-content-vec (->> filetxns
+                                     (mapv
+                                      #(when (is-journals-or-pages? %)
+                                         [% (db/get-file repo (config/get-file-path repo (relative-path %)))]))
+                                     (remove nil?))]
+        (<! (update-local-files rsapi graph-uuid base-path (map relative-path filetxns)))
+        (doseq [[filetxn origin-db-content] txn->db-content-vec]
+          (when (need-add-version-file? filetxn origin-db-content)
+            (add-new-version-file repo (relative-path filetxn) origin-db-content))))
+
+      (.-deleted? (first filetxns))
+      (let [filetxn (first filetxns)]
+        (assert (= 1 (count filetxns)))
         (let [r (<! (delete-local-files rsapi graph-uuid base-path [(relative-path filetxn)]))]
           (if (and (instance? ExceptionInfo r)
                    (string/index-of (str (ex-cause r)) "No such file or directory"))
@@ -928,7 +975,7 @@
   if local-txid != remote-txid, return {:need-sync-remote true}"))
 
 (defrecord Remote->LocalSyncer [user-uuid graph-uuid base-path repo *txid *sync-state
-                              ^:mutable local->remote-syncer *stopped]
+                                ^:mutable local->remote-syncer *stopped]
   Object
   (set-local->remote-syncer! [_ s] (set! local->remote-syncer s))
   (sync-files-remote->local!

src/main/frontend/handler/file_sync.cljs

@@ -1,7 +1,9 @@
 (ns frontend.handler.file-sync
   (:require ["path" :as path]
             [cljs-time.coerce :as tc]
+            [cljs-time.format :as tf]
             [cljs.core.async :as async :refer [go <!]]
+            [cljs.core.async.interop :refer [p->c]]
             [clojure.string :as string]
             [frontend.config :as config]
             [frontend.db :as db]
@@ -9,7 +11,8 @@
             [frontend.handler.notification :as notification]
             [frontend.state :as state]
             [frontend.util :as util]
-            [frontend.handler.user :as user]))
+            [frontend.handler.user :as user]
+            [frontend.fs :as fs]))
 
 (def hiding-login&file-sync (not config/dev?))
 (def refresh-file-sync-component (atom false))
@@ -65,31 +68,78 @@
         (notification/show! (ex-cause r) :error)
         (notification/show! [:div
                              [:div "Downloaded version file at: "]
-                             [:div key]] :success false)))))
+                             [:div key]] :success false))
+      (when-not (instance? ExceptionInfo r)
+        key))))
+
+(defn- list-file-local-versions
+  [page]
+  (go
+    (when-let [path (-> page :block/file :file/path)]
+      (let [base-path           (config/get-repo-dir (state/get-current-repo))
+            rel-path            (string/replace-first path base-path "")
+            version-files-dir   (->> (path/join "version-files/local" rel-path)
+                                     path/parse
+                                     (#(js->clj % :keywordize-keys true))
+                                     ((juxt :dir :name))
+                                     (apply path/join base-path))
+            version-file-paths* (<! (p->c (fs/readdir version-files-dir)))]
+        (when-not (instance? ExceptionInfo version-file-paths*)
+          (let [version-file-paths
+                (filterv
+                 ;; filter dir
+                 (fn [dir-or-file]
+                   (-> (path/parse dir-or-file)
+                       (js->clj :keywordize-keys true)
+                       :ext
+                       seq))
+                 (js->clj (<! (p->c (fs/readdir version-files-dir)))))]
+            (mapv
+             (fn [path]
+               (let [create-time
+                     (-> (path/parse path)
+                         (js->clj :keywordize-keys true)
+                         :name
+                         (#(tf/parse (tf/formatter "yyyy-MM-dd'T'HH_mm_ss.SSSZZ") %)))]
+                 {:create-time create-time :path path :relative-path (string/replace-first path base-path "")}))
+             version-file-paths)))))))
 
 (defn list-file-versions [graph-uuid page]
   (let [file-id (:db/id (:block/file page))]
     (when-let [path (:file/path (db/entity file-id))]
       (let [base-path (config/get-repo-dir (state/get-current-repo))
-            path* (string/replace-first path base-path "")]
+            path*     (string/replace-first path base-path "")]
         (go
-          (let [version-list (:VersionList
-                              (<! (sync/get-remote-file-versions sync/remoteapi graph-uuid path*)))]
+          (let [version-list       (:VersionList
+                                    (<! (sync/get-remote-file-versions sync/remoteapi graph-uuid path*)))
+                local-version-list (<! (list-file-local-versions page))
+                all-version-list   (->> (concat version-list local-version-list)
+                                        (sort-by #(or (tc/from-string (:CreateTime %))
+                                                      (:create-time %))
+                                                 >))]
             (notification/show! [:div
                                  [:div.font-bold "File history - " path*]
                                  [:hr.my-2]
-                                 (for [version version-list]
-                                   (let [version-uuid (:VersionUUID version)]
+                                 (for [version all-version-list]
+                                   (let [version-uuid (or (:VersionUUID version) (:relative-path version))
+                                         local?       (some? (:relative-path version))]
                                      [:div.my-4 {:key version-uuid}
                                       [:div
                                        [:a.text-xs.inline
-                                        {:on-click #(download-version-file graph-uuid
-                                                                           (:FileUUID version)
-                                                                           (:VersionUUID version))}
+                                        {:on-click #(if local?
+                                                      (js/window.apis.openPath (:path version))
+                                                      (go
+                                                        (let [relative-path
+                                                              (<! (download-version-file graph-uuid
+                                                                                         (:FileUUID version)
+                                                                                         (:VersionUUID version)))]
+                                                          (js/window.apis.openPath (path/join base-path relative-path)))))}
                                         version-uuid]
-                                       [:div.opacity-70 (str "Size: " (:Size version))]]
+                                       (when-not local?
+                                         [:div.opacity-70 (str "Size: " (:Size version))])]
                                       [:div.opacity-50
-                                       (util/time-ago (tc/from-string (:CreateTime version)))]]))]
+                                       (util/time-ago (or (tc/from-string (:CreateTime version))
+                                                          (:create-time version)))]]))]
                                 :success false)))))))
 
 (defn get-current-graph-uuid [] (second @sync/graphs-txid))