enhance(sync): sort files when remote->local-full-sync

src/main/frontend/components/sidebar.cljs

@@ -388,22 +388,6 @@
                        (:current-parsing-file state))]]]]
     (ui/progress-bar-with-label width left-label (str finished "/" total))))
 
-(rum/defc file-sync-download-progress < rum/static
-  [state]
-  (let [finished (or (:finished state) 0)
-        total (:total state)]
-    (if (= total :unknown)
-      (ui/loading "Loading...")
-      (let [width (js/Math.round (* (.toFixed (/ finished total) 2) 100))
-           left-label [:div.flex.flex-row.font-bold
-                       "Downloading"
-                       [:div.hidden.md:flex.flex-row
-                        [:span.mr-1 ": "]
-                        [:ul
-                         (for [file (:downloading-files state)]
-                           [:li file])]]]]
-       (ui/progress-bar-with-label width left-label (str finished "/" total))))))
-
 (rum/defc main-content < rum/reactive db-mixins/query
   {:init (fn [state]
            (when-not @sidebar-inited?
@@ -429,16 +413,8 @@
         loading-files? (when current-repo (state/sub [:repo/loading-files? current-repo]))
         journals-length (state/sub :journals-length)
         latest-journals (db/get-latest-journals (state/get-current-repo) journals-length)
-        graph-parsing-state (state/sub [:graph/parsing-state current-repo])
-        graph-file-sync-download-init-state (state/sub [:file-sync/download-init-progress current-repo])]
+        graph-parsing-state (state/sub [:graph/parsing-state current-repo])]
     (cond
-      (or
-       (:downloading? graph-file-sync-download-init-state)
-       (not= (:total graph-file-sync-download-init-state) (:finished graph-file-sync-download-init-state)))
-      [:div.flex.items-center.justify-center.full-height-without-header
-       [:div.flex-1
-        (file-sync-download-progress graph-file-sync-download-init-state)]]
-
       (or
        (:graph-loading? graph-parsing-state)
        (not= (:total graph-parsing-state) (:finished graph-parsing-state)))

src/main/frontend/fs/sync.cljs

@@ -1,6 +1,8 @@
 (ns frontend.fs.sync
   (:require [cljs-http.client :as http]
             [cljs-time.core :as t]
+            [cljs-time.format :as tf]
+            [cljs-time.coerce :as tc]
             [cljs.core.async :as async :refer [go timeout go-loop offer! poll! chan <! >!]]
             [cljs.core.async.impl.channels]
             [cljs.core.async.interop :refer [p->c]]
@@ -428,6 +430,40 @@
     :else
     (throw (js/Error. (str "unsupport type " (str o))))))
 
+(defn- sort-file-metatdata-fn
+  ":recent-days-range > :favorite-pages > small-size pages > ...
+  :recent-days-range : [<min-inst-ms> <max-inst-ms>]
+"
+  [& {:keys [recent-days-range favorite-pages]}]
+  {:pre [(or (nil? recent-days-range)
+             (every? number? recent-days-range))]}
+  (let [favorite-pages* (set favorite-pages)]
+    (fn [^FileMetadata item]
+      (let [path (relative-path item)
+            journal? (string/starts-with? path "journals/")
+            journal-day
+            (when journal?
+              (try
+                (tc/to-long
+                 (tf/parse (tf/formatter "yyyy_MM_dd")
+                           (-> path
+                               (string/replace-first "journals/" "")
+                               (string/replace-first ".md" ""))))
+                (catch :default _)))]
+        (cond
+          (and recent-days-range
+               journal-day
+               (<= (first recent-days-range)
+                   ^number journal-day
+                   (second recent-days-range)))
+          journal-day
+
+          (contains? favorite-pages* path)
+          (count path)
+
+          :else
+          (- (.-size item)))))))
+
 ;;; ### APIs
 ;; `RSAPI` call apis through rsapi package, supports operations on files
 
@@ -1078,9 +1114,8 @@
 
 (def full-sync-chan (chan 1))
 (def stop-sync-chan (chan 1))
-(def remote->local-sync-chan
-  "not used"
-  (chan 1))
+(def remote->local-sync-chan (chan 1))
+(def remote->local-full-sync-chan (chan 1))
 (def local->remote-sync-chan
   "not used"
   (chan))
@@ -1250,11 +1285,13 @@
             remote-all-files-meta (<! remote-all-files-meta-c)
             local-all-files-meta (<! local-all-files-meta-c)
             diff-remote-files (set/difference remote-all-files-meta local-all-files-meta)
+            recent-10-days-range ((juxt #(tc/to-long (t/minus % (t/days 10))) #(tc/to-long %)) (t/today))
+            sorted-diff-remote-files (sort-by (sort-file-metatdata-fn :recent-days-range recent-10-days-range) > diff-remote-files)
             latest-txid (:TXId (<! (get-remote-graph remoteapi nil graph-uuid)))]
         (println "[full-sync(remote->local)]"
-                 (count diff-remote-files) "files need to sync")
+                 (count sorted-diff-remote-files) "files need to sync")
         (<! (.sync-files-remote->local!
-             this (map relative-path diff-remote-files)
+             this (map relative-path sorted-diff-remote-files)
              latest-txid))))))
 
 (defn- file-changed?
@@ -1454,8 +1491,7 @@
 (defrecord ^:large-vars/cleanup-todo
     SyncManager [graph-uuid base-path *sync-state
                  ^Local->RemoteSyncer local->remote-syncer ^Remote->LocalSyncer remote->local-syncer remoteapi
-                 full-sync-chan stop-sync-chan remote->local-sync-chan local->remote-sync-chan
-                 local-changes-chan ^:mutable ratelimit-local-changes-chan
+                 ^:mutable ratelimit-local-changes-chan
                  *txid ^:mutable state ^:mutable _remote-change-chan ^:mutable _*ws *stopped?
                  ^:mutable ops-chan]
     Object
@@ -1487,9 +1523,10 @@
       (set! _remote-change-chan (ws-listen! graph-uuid _*ws))
       (set! ratelimit-local-changes-chan (ratelimit local->remote-syncer local-changes-chan))
       (go-loop []
-        (let [{:keys [stop remote->local local->remote-full-sync local->remote]}
+        (let [{:keys [stop remote->local remote->local-full-sync local->remote-full-sync local->remote]}
               (async/alt!
                 stop-sync-chan {:stop true}
+                remote->local-full-sync-chan {:remote->local-full-sync true}
                 remote->local-sync-chan {:remote->local true}
                 full-sync-chan {:local->remote-full-sync true}
                 _remote-change-chan ([v] (println "remote change:" v) {:remote->local v})
@@ -1500,6 +1537,10 @@
             stop
             (do (drain-chan ops-chan)
                 (>! ops-chan {:stop true}))
+            remote->local-full-sync
+            (do (drain-chan ops-chan)
+                (>! ops-chan {:remote->local-full-sync true})
+                (recur))
             remote->local
             (let [txid
                   (if (true? remote->local)
@@ -1628,8 +1669,7 @@
         (debug/pprint ["stop sync-manager, graph-uuid" graph-uuid "base-path" base-path])
         (swap! *sync-state sync-state--update-state ::stop))))
 
-(defn sync-manager [user-uuid graph-uuid base-path repo txid *sync-state full-sync-chan stop-sync-chan
-                    remote->local-sync-chan local->remote-sync-chan local-changes-chan]
+(defn sync-manager [user-uuid graph-uuid base-path repo txid *sync-state]
   (let [*txid (atom txid)
         *stopped? (volatile! false)
         remoteapi-with-stop (->RemoteAPI *stopped?)
@@ -1644,8 +1684,7 @@
     (.set-remote->local-syncer! local->remote-syncer remote->local-syncer)
     (.set-local->remote-syncer! remote->local-syncer local->remote-syncer)
     (->SyncManager graph-uuid base-path *sync-state local->remote-syncer remote->local-syncer remoteapi-with-stop
-                   full-sync-chan stop-sync-chan
-                   remote->local-sync-chan local->remote-sync-chan local-changes-chan nil *txid nil nil nil *stopped? nil)))
+                   nil *txid nil nil nil *stopped? nil)))
 
 
 (defn sync-stop []
@@ -1686,8 +1725,7 @@
         repo (state/get-current-repo)
         sm (sync-manager current-user-uuid graph-uuid
                          (config/get-repo-dir repo) repo
-                         txid *sync-state full-sync-chan stop-sync-chan remote->local-sync-chan local->remote-sync-chan
-                         local-changes-chan)]
+                         txid *sync-state)]
     (when-not (config/demo-graph? repo)
       (go
        ;; 1. if remote graph has been deleted, clear graphs-txid.edn
@@ -1713,6 +1751,7 @@
                (drain-chan local-changes-chan)
                (poll! stop-sync-chan)
                (poll! remote->local-sync-chan)
+               (poll! remote->local-full-sync-chan)
 
                ;; update global state when *sync-state changes
                (add-watch *sync-state ::update-global-state
@@ -1721,7 +1760,9 @@
                (.start sm)
 
 
-               (offer! remote->local-sync-chan true)
+               (if (zero? txid)
+                 (offer! remote->local-full-sync-chan true)
+                 (offer! remote->local-sync-chan true))
                (offer! full-sync-chan true)
 
                ;; watch :network/online?

src/main/frontend/handler/file_sync.cljs

@@ -55,38 +55,6 @@
   []
   (go (:Graphs (<! (sync/list-remote-graphs sync/remoteapi)))))
 
-(defn download-all-files
-  [repo graph-uuid user-uuid base-path]
-  (go
-    (<! (sync/set-env sync/rsapi config/FILE-SYNC-PROD? nil nil))
-    (<! (sync/ensure-pwd+keys-exists! graph-uuid repo))
-    (<! (sync/set-env sync/rsapi config/FILE-SYNC-PROD?
-                      (get-in @sync/pwd-map [graph-uuid :private-key])
-                      (get-in @sync/pwd-map [graph-uuid :private-key])))
-    (state/reset-file-sync-download-init-state!)
-    (state/set-file-sync-download-init-state! {:total :unknown :finished 0 :downloading? true})
-    (let [remote-all-files-meta (<! (sync/get-remote-all-files-meta sync/remoteapi graph-uuid))
-          local-all-files-meta (<! (sync/get-local-all-files-meta sync/rsapi graph-uuid base-path))
-          diff-remote-files (set/difference remote-all-files-meta local-all-files-meta)
-          latest-txid (:TXId (<! (sync/get-remote-graph sync/remoteapi nil graph-uuid)))
-          partitioned-filetxns
-          (sequence (sync/filepaths->partitioned-filetxns 10 graph-uuid user-uuid)
-                    (map sync/relative-path diff-remote-files))]
-      (state/set-file-sync-download-init-state! {:total (count diff-remote-files) :finished 0})
-      (let [r (<! (sync/apply-filetxns-partitions
-                   nil user-uuid graph-uuid base-path partitioned-filetxns repo nil (atom false)
-                   (fn [filetxns]
-                     (state/set-file-sync-download-init-state!
-                      {:downloading-files (mapv sync/relative-path filetxns)}))
-                   (fn [filetxns]
-                     (state/set-file-sync-download-init-state!
-                      {:finished (+ (count filetxns)
-                                    (or (:finished (state/get-file-sync-download-init-state)) 0))}))))]
-        (if (instance? ExceptionInfo r)
-          ;; TODO: add re-download button
-          (notification/show! (str "Download graph failed: " (ex-cause r)) :warning)
-          (do (state/reset-file-sync-download-init-state!)
-              (<! (sync/update-graphs-txid! latest-txid graph-uuid user-uuid repo))))))))
 
 (defn load-session-graphs
   []
@@ -101,14 +69,10 @@
 
 (defn init-graph [graph-uuid]
   (let [repo (state/get-current-repo)
-        base-path (config/get-repo-dir repo)
         user-uuid (user/user-uuid)]
     (sync/update-graphs-txid! 0 graph-uuid user-uuid repo)
-    (go (sync/sync-stop)
-        (<! (download-all-files repo graph-uuid user-uuid base-path))
-        (println :debug sync/graphs-txid)
-        (swap! refresh-file-sync-component not)
-        (state/pub-event! [:graph/switch repo {:persist? false}]))))
+    (swap! refresh-file-sync-component not)
+    (state/pub-event! [:graph/switch repo {:persist? false}])))
 
 (defn- download-version-file [graph-uuid file-uuid version-uuid]
   (go

src/main/frontend/state.cljs

@@ -229,8 +229,6 @@
      :file-sync/sync-uploading-files        nil
      :file-sync/sync-downloading-files      nil
 
-     :file-sync/download-init-progress      nil
-
      :encryption/graph-parsing?             false
      })))
 
@@ -1678,20 +1676,6 @@
 (defn get-file-sync-state [repo]
   (get-in @state [:file-sync/sync-state repo]))
 
-(defn reset-file-sync-download-init-state!
-  []
-  (set-state! [:file-sync/download-init-progress (get-current-repo)] {}))
-
-(defn set-file-sync-download-init-state!
-  [m]
-  (update-state! [:file-sync/download-init-progress (get-current-repo)]
-                 (if (fn? m) m
-                     (fn [old-value] (merge old-value m)))))
-
-(defn get-file-sync-download-init-state
-  []
-  (get-in @state [:file-sync/download-init-progress (get-current-repo)]))
-
 (defn reset-parsing-state!
   []
   (set-state! [:graph/parsing-state (get-current-repo)] {}))

yarn.lock

@@ -6832,6 +6832,11 @@ [email protected]:
     react-draggable "3.x"
     react-resizable "1.x"
 
[email protected]:
+  version "2.1.0"
+  resolved "https://registry.npmmirror.com/react-icon-base/-/react-icon-base-2.1.0.tgz#a196e33fdf1e7aaa1fda3aefbb68bdad9e82a79d"
+  integrity sha512-9wwKJa2LB8ujtJB5MAXYYEM7JfYThZTj0YnfGxzLLWkifaLIGc7iTde2EpJ7ka5MjneRHnlxbIn5VV9k2WjUVA==
+
 react-icon-base@^2.1.2:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/react-icon-base/-/react-icon-base-2.1.2.tgz#a17101dad9c1192652356096860a9ab43a0766c7"