enhance(sync): add progress-bar when init-download remote graph

src/main/frontend/components/sidebar.cljs

@@ -341,6 +341,20 @@
                        (:current-parsing-file state))]]]]
     (ui/progress-bar-with-label width left-label (str finished "/" total))))
 
+(rum/defc file-sync-download-progress < rum/static
+  [state]
+  (let [finished (or (:finished state) 0)
+        total (:total state)
+        width (js/Math.round (* (.toFixed (/ finished total) 2) 100))
+        left-label [:div.flex.flex-row.font-bold
+                    "Downloading"
+                    [:div.hidden.md:flex.flex-row
+                     [:span.mr-1 ": "]
+                     [:ul
+                      (for [file (:downloading-files state)]
+                        [:li file])]]]]
+    (ui/progress-bar-with-label width left-label (str finished "/" total))))
+
 (rum/defc main-content < rum/reactive db-mixins/query
   {:init (fn [state]
            (when-not @sidebar-inited?
@@ -364,8 +378,16 @@
         loading-files? (when current-repo (state/sub [:repo/loading-files? current-repo]))
         journals-length (state/sub :journals-length)
         latest-journals (db/get-latest-journals (state/get-current-repo) journals-length)
-        graph-parsing-state (state/sub [:graph/parsing-state current-repo])]
+        graph-parsing-state (state/sub [:graph/parsing-state current-repo])
+        graph-file-sync-download-init-state (state/sub [:file-sync/download-init-progress current-repo])]
     (cond
+      (or
+       (:downloading? graph-file-sync-download-init-state)
+       (not= (:total graph-file-sync-download-init-state) (:finished graph-file-sync-download-init-state)))
+      [:div.flex.items-center.justify-center.full-height-without-header
+       [:div.flex-1
+        (file-sync-download-progress graph-file-sync-download-init-state)]]
+
       (or
        (:graph-loading? graph-parsing-state)
        (not= (:total graph-parsing-state) (:finished graph-parsing-state)))

src/main/frontend/fs/sync.cljs

@@ -353,7 +353,7 @@
    :TXType "update_files"
    :TXContent (string/join "/" [user-uuid graph-uuid relative-path])})
 
-(defn- filepaths->partitioned-filetxns
+(defn filepaths->partitioned-filetxns
   "transducer.
   1. filepaths -> diff
   2. diffs->partitioned-filetxns"
@@ -364,6 +364,7 @@
    (map-indexed filepath->diff)
    (diffs->partitioned-filetxns n)))
 
+
 (deftype FileMetadata [size etag path last-modified remote? ^:mutable normalized-path]
   Object
   (get-normalized-path [_]
@@ -390,7 +391,7 @@
   (-pr-writer [_ w _opts]
     (write-all w (str {:size size :etag etag :path path :remote? remote?}))))
 
-(defn- relative-path [o]
+(defn relative-path [o]
   (cond
     (implements? IRelativePath o)
     (-relative-path o)
@@ -754,8 +755,9 @@
             repo (state/get-current-repo)
             file-path (config/get-file-path repo path)
             content (<! (p->c (fs/read-file "" file-path)))]
-        (or (nil? content)
-            (some :removed (diff/diff origin-db-content content)))))))
+        (and origin-db-content
+             (or (nil? content)
+                 (some :removed (diff/diff origin-db-content content))))))))
 
 (defn- apply-filetxns
   [graph-uuid base-path filetxns]
@@ -797,21 +799,23 @@
          sync-state--remove-current-remote->local-files
          sync-state--stopped?)
 
-(defn- apply-filetxns-partitions
+(defn apply-filetxns-partitions
   "won't call update-graph-txid! when *txid is nil"
-  [*sync-state user-uuid graph-uuid base-path filetxns-partitions repo *txid *stopped]
+  [*sync-state user-uuid graph-uuid base-path filetxns-partitions repo *txid *stopped before-f after-f]
   (go-loop [filetxns-partitions* filetxns-partitions]
     (if @*stopped
       {:stop true}
       (when (seq filetxns-partitions*)
         (let [filetxns (first filetxns-partitions*)
               paths (map relative-path filetxns)
-              _ (swap! *sync-state sync-state--add-current-remote->local-files paths)
+              _ (when (and before-f (fn? before-f)) (before-f filetxns))
+              _ (when *sync-state (swap! *sync-state sync-state--add-current-remote->local-files paths))
               r (<! (apply-filetxns graph-uuid base-path filetxns))
-              _ (swap! *sync-state sync-state--remove-current-remote->local-files paths)]
+              _ (when *sync-state (swap! *sync-state sync-state--remove-current-remote->local-files paths))]
           (if (instance? ExceptionInfo r)
             r
             (let [latest-txid (apply max (map #(.-txid ^FileTxn %) filetxns))]
+              (when (and after-f (fn? after-f)) (after-f filetxns))
               (when *txid
                 (reset! *txid latest-txid)
                 (update-graphs-txid! latest-txid graph-uuid user-uuid repo))
@@ -982,25 +986,25 @@
     [_ relative-filepaths latest-txid]
     (go
       (let [partitioned-filetxns
-              (sequence (filepaths->partitioned-filetxns 10 graph-uuid user-uuid)
-                        relative-filepaths)
-              r
-              (if (empty? (flatten partitioned-filetxns))
-                {:succ true}
-                (<! (apply-filetxns-partitions
-                     *sync-state user-uuid graph-uuid base-path partitioned-filetxns repo
-                     nil *stopped)))]
-          (cond
-            (instance? ExceptionInfo r)
-            {:unknown r}
+            (sequence (filepaths->partitioned-filetxns 10 graph-uuid user-uuid)
+                      relative-filepaths)
+            r
+            (if (empty? (flatten partitioned-filetxns))
+              {:succ true}
+              (<! (apply-filetxns-partitions
+                   *sync-state user-uuid graph-uuid base-path partitioned-filetxns repo
+                   nil *stopped nil nil)))]
+        (cond
+          (instance? ExceptionInfo r)
+          {:unknown r}
 
-            @*stopped
-            {:stop true}
+          @*stopped
+          {:stop true}
 
-            :else
-            (do (update-graphs-txid! latest-txid graph-uuid user-uuid repo)
-                (reset! *txid latest-txid)
-                {:succ true})))))
+          :else
+          (do (update-graphs-txid! latest-txid graph-uuid user-uuid repo)
+              (reset! *txid latest-txid)
+              {:succ true})))))
 
   IRemote->LocalSync
   (stop-remote->local! [_] (vreset! *stopped true))
@@ -1029,7 +1033,8 @@
                               (reset! *txid latest-txid)
                               {:succ true})
                           (<! (apply-filetxns-partitions
-                               *sync-state user-uuid graph-uuid base-path partitioned-filetxns repo *txid *stopped)))))))))]
+                               *sync-state user-uuid graph-uuid base-path
+                               partitioned-filetxns repo *txid *stopped nil nil)))))))))]
         (cond
           (instance? ExceptionInfo r)
           {:unknown r}
@@ -1050,12 +1055,11 @@
             remote-all-files-meta (<! remote-all-files-meta-c)
             local-all-files-meta (<! local-all-files-meta-c)
             diff-remote-files (set/difference remote-all-files-meta local-all-files-meta)
-            latest-txid (:TXId
-                         (<! (get-remote-graph remoteapi nil graph-uuid)))]
+            latest-txid (:TXId (<! (get-remote-graph remoteapi nil graph-uuid)))]
         (println "[full-sync(remote->local)]"
                  (count diff-remote-files) "files need to sync")
         (<! (.sync-files-remote->local!
-             this (map -relative-path diff-remote-files)
+             this (map relative-path diff-remote-files)
              latest-txid))))))
 
 (defn- file-changed?

src/main/frontend/handler/file_sync.cljs

@@ -5,6 +5,7 @@
             [cljs.core.async :as async :refer [go <!]]
             [cljs.core.async.interop :refer [p->c]]
             [clojure.string :as string]
+            [clojure.set :as set]
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.fs.sync :as sync]
@@ -54,9 +55,41 @@
   []
   (go (:Graphs (<! (sync/list-remote-graphs sync/remoteapi)))))
 
+(defn download-all-files
+  [repo graph-uuid user-uuid base-path]
+  (go
+    (state/reset-file-sync-download-init-state!)
+    (state/set-file-sync-download-init-state! {:total js/NaN :finished 0 :downloading? true})
+    (let [remote-all-files-meta (<! (sync/get-remote-all-files-meta sync/remoteapi graph-uuid))
+          local-all-files-meta (<! (sync/get-local-all-files-meta sync/rsapi graph-uuid base-path))
+          diff-remote-files (set/difference remote-all-files-meta local-all-files-meta)
+          latest-txid (:TXId (<! (sync/get-remote-graph sync/remoteapi nil graph-uuid)))
+          partitioned-filetxns
+          (sequence (sync/filepaths->partitioned-filetxns 10 graph-uuid user-uuid)
+                    (map sync/relative-path diff-remote-files))]
+      (state/set-file-sync-download-init-state! {:total (count diff-remote-files) :finished 0})
+      (let [r (<! (sync/apply-filetxns-partitions
+                   nil user-uuid graph-uuid base-path partitioned-filetxns repo nil (atom false)
+                   (fn [filetxns]
+                     (state/set-file-sync-download-init-state!
+                      {:downloading-files (mapv sync/relative-path filetxns)}))
+                   (fn [filetxns]
+                     (state/set-file-sync-download-init-state!
+                      {:finished (+ (count filetxns)
+                                    (or (:finished (state/get-file-sync-download-init-state)) 0))}))))]
+        (if (instance? ExceptionInfo r)
+          ;; TODO: add re-download button
+          (notification/show! (str "Download graph failed: " (ex-cause r)) :warning)
+          (do (state/reset-file-sync-download-init-state!)
+              (sync/update-graphs-txid! latest-txid graph-uuid user-uuid repo)))))))
+
 (defn switch-graph [graph-uuid]
-  (sync/update-graphs-txid! 0 graph-uuid (user/user-uuid) (state/get-current-repo))
-  (swap! refresh-file-sync-component not))
+  (let [repo (state/get-current-repo)
+        base-path (config/get-repo-dir repo)
+        user-uuid (user/user-uuid)]
+    (sync/update-graphs-txid! 0 graph-uuid user-uuid repo)
+    (download-all-files repo graph-uuid user-uuid base-path)
+    (swap! refresh-file-sync-component not)))
 
 (defn- download-version-file [graph-uuid file-uuid version-uuid]
 
@@ -143,3 +176,5 @@
                                 :success false)))))))
 
 (defn get-current-graph-uuid [] (second @sync/graphs-txid))
+
+(defn get-current-graph-txid [] (nth @sync/graphs-txid 2 nil))

src/main/frontend/state.cljs

@@ -218,6 +218,8 @@
      :file-sync/sync-uploading-files        nil
      :file-sync/sync-downloading-files      nil
 
+     :file-sync/download-init-progress      nil
+
      :encryption/graph-parsing?             false
      })))
 
@@ -1671,6 +1673,20 @@
 (defn get-file-sync-state []
   (:file-sync/sync-state @state))
 
+(defn reset-file-sync-download-init-state!
+  []
+  (set-state! [:file-sync/download-init-progress (get-current-repo)] {}))
+
+(defn set-file-sync-download-init-state!
+  [m]
+  (update-state! [:file-sync/download-init-progress (get-current-repo)]
+                 (if (fn? m) m
+                     (fn [old-value] (merge old-value m)))))
+
+(defn get-file-sync-download-init-state
+  []
+  (get-in @state [:file-sync/download-init-progress (get-current-repo)]))
+
 (defn reset-parsing-state!
   []
   (set-state! [:graph/parsing-state (get-current-repo)] {}))