|
|
@@ -1,7 +1,6 @@
|
|
|
(ns frontend.fs.sync
|
|
|
"Main ns for providing file sync functionality"
|
|
|
- (:require ["@capawesome/capacitor-background-task" :refer [BackgroundTask]]
|
|
|
- ["path" :as node-path]
|
|
|
+ (:require ["path" :as node-path]
|
|
|
[cljs-http.client :as http]
|
|
|
[cljs-time.coerce :as tc]
|
|
|
[cljs-time.core :as t]
|
|
|
@@ -23,12 +22,10 @@
|
|
|
[frontend.diff :as diff]
|
|
|
[frontend.encrypt :as encrypt]
|
|
|
[frontend.fs :as fs]
|
|
|
- [frontend.fs.capacitor-fs :as capacitor-fs]
|
|
|
[frontend.fs.diff-merge :as diff-merge]
|
|
|
[frontend.handler.file-based.file :as file-handler]
|
|
|
[frontend.handler.notification :as notification]
|
|
|
[frontend.handler.user :as user]
|
|
|
- [frontend.mobile.util :as mobile-util]
|
|
|
[frontend.pubsub :as pubsub]
|
|
|
[frontend.state :as state]
|
|
|
[frontend.util :as util]
|
|
|
@@ -164,8 +161,7 @@
|
|
|
:remote->local-full-sync-failed
|
|
|
:local->remote-full-sync-failed
|
|
|
:get-remote-graph-failed
|
|
|
- :get-deletion-logs-failed
|
|
|
- })
|
|
|
+ :get-deletion-logs-failed})
|
|
|
|
|
|
(s/def ::sync-event (s/keys :req-un [::event ::data]))
|
|
|
|
|
|
@@ -198,7 +194,6 @@
|
|
|
[x]
|
|
|
(when (instance? ExceptionInfo x) x))
|
|
|
|
|
|
-
|
|
|
(def ws-addr config/WS-URL)
|
|
|
|
|
|
;; Warning: make sure to `persist-var/-load` graphs-txid before using it.
|
|
|
@@ -326,9 +321,9 @@
|
|
|
(defn <request [api-name & args]
|
|
|
(let [name (str api-name (.now js/Date))]
|
|
|
(go (swap! *on-flying-request conj name)
|
|
|
- (let [r (<! (apply <request* api-name args))]
|
|
|
- (swap! *on-flying-request disj name)
|
|
|
- r))))
|
|
|
+ (let [r (<! (apply <request* api-name args))]
|
|
|
+ (swap! *on-flying-request disj name)
|
|
|
+ r))))
|
|
|
|
|
|
(defn- remove-dir-prefix [dir path]
|
|
|
(let [r (string/replace path (js/RegExp. (str "^" (gstring/regExpEscape dir))) "")]
|
|
|
@@ -417,19 +412,19 @@
|
|
|
(let [update? (= "update_files" TXType)
|
|
|
delete? (= "delete_files" TXType)
|
|
|
update-xf
|
|
|
- (comp
|
|
|
- (remove #(or (empty? (first %))
|
|
|
- (empty? (last %))))
|
|
|
- (map #(->FileTxn (first %) (first %) update? delete? TXId (last %))))
|
|
|
+ (comp
|
|
|
+ (remove #(or (empty? (first %))
|
|
|
+ (empty? (last %))))
|
|
|
+ (map #(->FileTxn (first %) (first %) update? delete? TXId (last %))))
|
|
|
delete-xf
|
|
|
- (comp
|
|
|
- (remove #(empty? (first %)))
|
|
|
- (map #(->FileTxn (first %) (first %) update? delete? TXId nil)))
|
|
|
+ (comp
|
|
|
+ (remove #(empty? (first %)))
|
|
|
+ (map #(->FileTxn (first %) (first %) update? delete? TXId nil)))
|
|
|
rename-xf
|
|
|
- (comp
|
|
|
- (remove #(or (empty? (first %))
|
|
|
- (empty? (second %))))
|
|
|
- (map #(->FileTxn (second %) (first %) false false TXId nil)))
|
|
|
+ (comp
|
|
|
+ (remove #(or (empty? (first %))
|
|
|
+ (empty? (second %))))
|
|
|
+ (map #(->FileTxn (second %) (first %) false false TXId nil)))
|
|
|
xf (case TXType
|
|
|
"delete_files" delete-xf
|
|
|
"update_files" update-xf
|
|
|
@@ -560,7 +555,6 @@
|
|
|
(map-indexed filepath+checksum->diff)
|
|
|
(diffs->partitioned-filetxns n)))
|
|
|
|
|
|
-
|
|
|
(deftype FileMetadata [size etag path encrypted-path last-modified remote? txid ^:mutable normalized-path]
|
|
|
Object
|
|
|
(get-normalized-path [_]
|
|
|
@@ -596,13 +590,10 @@
|
|
|
:txid txid
|
|
|
not-found))
|
|
|
|
|
|
-
|
|
|
IPrintWithWriter
|
|
|
(-pr-writer [_ w _opts]
|
|
|
(write-all w (str {:size size :etag etag :path path :remote? remote? :txid txid :last-modified last-modified}))))
|
|
|
|
|
|
-
|
|
|
-
|
|
|
(def ^:private higher-priority-remote-files
|
|
|
"when diff all remote files and local files, following remote files always need to download(when checksum not matched),
|
|
|
even local-file's last-modified > remote-file's last-modified.
|
|
|
@@ -659,21 +650,21 @@
|
|
|
#{} s1))
|
|
|
|
|
|
(comment
|
|
|
- (defn map->FileMetadata [m]
|
|
|
- (apply ->FileMetadata ((juxt :size :etag :path :encrypted-path :last-modified :remote? (constantly nil)) m)))
|
|
|
-
|
|
|
- (assert
|
|
|
- (=
|
|
|
- #{(map->FileMetadata {:size 1 :etag 2 :path 2 :encrypted-path 2 :last-modified 2})}
|
|
|
- (diff-file-metadata-sets
|
|
|
- (into #{}
|
|
|
- (map map->FileMetadata)
|
|
|
- [{:size 1 :etag 1 :path 1 :encrypted-path 1 :last-modified 1}
|
|
|
- {:size 1 :etag 2 :path 2 :encrypted-path 2 :last-modified 2}])
|
|
|
- (into #{}
|
|
|
- (map map->FileMetadata)
|
|
|
- [{:size 1 :etag 1 :path 1 :encrypted-path 1 :last-modified 1}
|
|
|
- {:size 1 :etag 1 :path 2 :encrypted-path 2 :last-modified 1}])))))
|
|
|
+ (defn map->FileMetadata [m]
|
|
|
+ (apply ->FileMetadata ((juxt :size :etag :path :encrypted-path :last-modified :remote? (constantly nil)) m)))
|
|
|
+
|
|
|
+ (assert
|
|
|
+ (=
|
|
|
+ #{(map->FileMetadata {:size 1 :etag 2 :path 2 :encrypted-path 2 :last-modified 2})}
|
|
|
+ (diff-file-metadata-sets
|
|
|
+ (into #{}
|
|
|
+ (map map->FileMetadata)
|
|
|
+ [{:size 1 :etag 1 :path 1 :encrypted-path 1 :last-modified 1}
|
|
|
+ {:size 1 :etag 2 :path 2 :encrypted-path 2 :last-modified 2}])
|
|
|
+ (into #{}
|
|
|
+ (map map->FileMetadata)
|
|
|
+ [{:size 1 :etag 1 :path 1 :encrypted-path 1 :last-modified 1}
|
|
|
+ {:size 1 :etag 1 :path 2 :encrypted-path 2 :last-modified 1}])))))
|
|
|
|
|
|
(extend-protocol IChecksum
|
|
|
FileMetadata
|
|
|
@@ -726,7 +717,6 @@
|
|
|
|
|
|
common-util/path-normalize)
|
|
|
|
|
|
-
|
|
|
;;; ### APIs
|
|
|
;; `RSAPI` call apis through rsapi package, supports operations on files
|
|
|
|
|
|
@@ -765,16 +755,13 @@
|
|
|
(<get-graph-encrypt-keys [this graph-uuid])
|
|
|
(<upload-graph-encrypt-keys [this graph-uuid public-key encrypted-private-key]))
|
|
|
|
|
|
-
|
|
|
(defprotocol IRemoteControlAPI
|
|
|
"api functions provided for outside the sync process"
|
|
|
- (<delete-remote-files-control [this graph-uuid filepaths])
|
|
|
- )
|
|
|
+ (<delete-remote-files-control [this graph-uuid filepaths]))
|
|
|
|
|
|
(defprotocol IToken
|
|
|
(<get-token [this]))
|
|
|
|
|
|
-
|
|
|
(defn <case-different-local-file-exist?
|
|
|
"e.g. filepath=\"pages/Foo.md\"
|
|
|
found-filepath=\"pages/foo.md\"
|
|
|
@@ -921,149 +908,10 @@
|
|
|
(<add-new-version [_this repo path content]
|
|
|
(p->c (ipc/ipc "addVersionFile" (config/get-local-dir repo) path content))))
|
|
|
|
|
|
-(deftype ^:large-vars/cleanup-todo CapacitorAPI [^:mutable graph-uuid' ^:mutable private-key ^:mutable public-key']
|
|
|
- IToken
|
|
|
- (<get-token [_this]
|
|
|
- (user/<wrap-ensure-id&access-token
|
|
|
- (state/get-auth-id-token)))
|
|
|
-
|
|
|
- IRSAPI
|
|
|
- (rsapi-ready? [_ graph-uuid] (and (= graph-uuid graph-uuid') private-key public-key'))
|
|
|
- (<key-gen [_]
|
|
|
- (go (let [r (<! (p->c (.keygen mobile-util/file-sync #js {})))]
|
|
|
- (-> r
|
|
|
- (js->clj :keywordize-keys true)))))
|
|
|
- (<set-env [_ graph-uuid prod? secret-key public-key]
|
|
|
- (set! graph-uuid' graph-uuid)
|
|
|
- (set! private-key secret-key)
|
|
|
- (set! public-key' public-key)
|
|
|
- (p->c (.setEnv mobile-util/file-sync (clj->js {:graphUUID graph-uuid
|
|
|
- :env (if prod? "prod" "dev")
|
|
|
- :secretKey secret-key
|
|
|
- :publicKey public-key}))))
|
|
|
-
|
|
|
- (<get-local-all-files-meta [this graph-uuid base-path]
|
|
|
- (go
|
|
|
- (let [r (<! (p->c (.getLocalAllFilesMeta mobile-util/file-sync (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path}))))]
|
|
|
- (or (guard-ex r)
|
|
|
- (<! (<build-local-file-metadatas this graph-uuid (.-result r)))))))
|
|
|
-
|
|
|
- (<get-local-files-meta [this graph-uuid base-path filepaths]
|
|
|
- (go
|
|
|
- (let [r (<! (p->c (.getLocalFilesMeta mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :filePaths filepaths}))))]
|
|
|
- (assert (not (instance? ExceptionInfo r)) "get-local-files-meta shouldn't return exception")
|
|
|
- (<! (<build-local-file-metadatas this graph-uuid (.-result r))))))
|
|
|
-
|
|
|
- (<rename-local-file [_ graph-uuid base-path from to]
|
|
|
- (p->c (.renameLocalFile mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :from (path-normalize from)
|
|
|
- :to (path-normalize to)}))))
|
|
|
-
|
|
|
- (<update-local-files [this graph-uuid base-path filepaths]
|
|
|
- (go
|
|
|
- (let [token-or-exp (<! (<get-token this))
|
|
|
- filepaths' (map path-normalize filepaths)]
|
|
|
- (or (guard-ex token-or-exp)
|
|
|
- (<! (p->c (.updateLocalFiles mobile-util/file-sync (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :filePaths filepaths'
|
|
|
- :token token-or-exp}))))))))
|
|
|
- (<fetch-remote-files [this graph-uuid base-path filepaths]
|
|
|
- (go
|
|
|
- (let [token-or-exp (<! (<get-token this))]
|
|
|
- (or (guard-ex token-or-exp)
|
|
|
- (js->clj
|
|
|
- (.-value
|
|
|
- (<! (<retry-rsapi
|
|
|
- #(p->c (.fetchRemoteFiles mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :filePaths filepaths
|
|
|
- :token token-or-exp})))))))))))
|
|
|
- (<download-version-files [this graph-uuid base-path filepaths]
|
|
|
- (go
|
|
|
- (let [token-or-exp (<! (<get-token this))]
|
|
|
- (or (guard-ex token-or-exp)
|
|
|
- (<! (<retry-rsapi
|
|
|
- #(p->c (.updateLocalVersionFiles mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :filePaths filepaths
|
|
|
- :token token-or-exp})))))))))
|
|
|
-
|
|
|
- (<delete-local-files [_ graph-uuid base-path filepaths]
|
|
|
- (let [normalized-filepaths (mapv path-normalize filepaths)]
|
|
|
- (go
|
|
|
- (let [r (<! (<retry-rsapi #(p->c (.deleteLocalFiles mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :filePaths normalized-filepaths})))))]
|
|
|
- r))))
|
|
|
-
|
|
|
- (<update-remote-files [this graph-uuid base-path filepaths local-txid]
|
|
|
- (let [normalized-filepaths (mapv path-normalize filepaths)]
|
|
|
- (go
|
|
|
- (let [token-or-exp (<! (<get-token this))
|
|
|
- r (or (guard-ex token-or-exp)
|
|
|
- (<! (p->c (.updateRemoteFiles mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :filePaths normalized-filepaths
|
|
|
- :txid local-txid
|
|
|
- :token token-or-exp
|
|
|
- :fnameEncryption true})))))]
|
|
|
- (or (guard-ex r)
|
|
|
- (get (js->clj r) "txid"))))))
|
|
|
-
|
|
|
- (<delete-remote-files [this graph-uuid base-path filepaths local-txid]
|
|
|
- (let [normalized-filepaths (mapv path-normalize filepaths)]
|
|
|
- (go
|
|
|
- (let [token-or-exp (<! (<get-token this))
|
|
|
- r (or (guard-ex token-or-exp)
|
|
|
- (<! (p->c (.deleteRemoteFiles mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :basePath base-path
|
|
|
- :filePaths normalized-filepaths
|
|
|
- :txid local-txid
|
|
|
- :token token-or-exp})))))]
|
|
|
- (or (guard-ex r)
|
|
|
- (get (js->clj r) "txid"))))))
|
|
|
-
|
|
|
- (<encrypt-fnames [_ graph-uuid fnames]
|
|
|
- (go
|
|
|
- (let [r (<! (p->c (.encryptFnames mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :filePaths fnames}))))]
|
|
|
- (or (guard-ex r)
|
|
|
- (get (js->clj r) "value")))))
|
|
|
- (<decrypt-fnames [_ graph-uuid fnames]
|
|
|
- (go (let [r (<! (p->c (.decryptFnames mobile-util/file-sync
|
|
|
- (clj->js {:graphUUID graph-uuid
|
|
|
- :filePaths fnames}))))]
|
|
|
- (if (instance? ExceptionInfo r)
|
|
|
- (ex-info "decrypt-failed" {:fnames fnames} (ex-cause r))
|
|
|
- (get (js->clj r) "value")))))
|
|
|
- (<cancel-all-requests [_]
|
|
|
- (p->c (.cancelAllRequests mobile-util/file-sync)))
|
|
|
- (<add-new-version [_this repo path content]
|
|
|
- (p->c (capacitor-fs/backup-file repo :version-file-dir path content))))
|
|
|
-
|
|
|
(def rsapi (cond
|
|
|
(util/electron?)
|
|
|
(->RSAPI nil nil nil)
|
|
|
|
|
|
- (mobile-util/native-ios?)
|
|
|
- (->CapacitorAPI nil nil nil)
|
|
|
-
|
|
|
- (mobile-util/native-android?)
|
|
|
- (->CapacitorAPI nil nil nil)
|
|
|
-
|
|
|
:else
|
|
|
nil))
|
|
|
|
|
|
@@ -1128,8 +976,8 @@
|
|
|
(go-loop []
|
|
|
(let [{:keys [val stop]}
|
|
|
(async/alt!
|
|
|
- debug-print-sync-events-loop-stop-chan {:stop true}
|
|
|
- out-ch ([v] {:val v}))]
|
|
|
+ debug-print-sync-events-loop-stop-chan {:stop true}
|
|
|
+ out-ch ([v] {:val v}))]
|
|
|
(cond
|
|
|
stop (do (async/unmix-all out-mix)
|
|
|
(doseq [[topic ch] topic&chs]
|
|
|
@@ -1138,12 +986,10 @@
|
|
|
val (do (pp/pprint [:debug :sync-event val])
|
|
|
(recur))))))))
|
|
|
|
|
|
-
|
|
|
(defn stop-debug-print-sync-events-loop
|
|
|
[]
|
|
|
(offer! debug-print-sync-events-loop-stop-chan true))
|
|
|
|
|
|
-
|
|
|
;;; sync events ends
|
|
|
|
|
|
(defn- fire-file-sync-storage-exceed-limit-event!
|
|
|
@@ -1232,7 +1078,6 @@
|
|
|
(recur result-file-meta-list others))))
|
|
|
(vals (persistent! result-file-meta-list))))))
|
|
|
|
|
|
-
|
|
|
(extend-type RemoteAPI
|
|
|
IRemoteAPI
|
|
|
(<user-info [this]
|
|
|
@@ -1438,13 +1283,10 @@
|
|
|
|
|
|
(comment
|
|
|
(declare remoteapi)
|
|
|
- (<delete-remote-files-control remoteapi (second @graphs-txid) ["pages/aa.md"])
|
|
|
-
|
|
|
- )
|
|
|
+ (<delete-remote-files-control remoteapi (second @graphs-txid) ["pages/aa.md"]))
|
|
|
|
|
|
(def remoteapi (->RemoteAPI nil))
|
|
|
|
|
|
-
|
|
|
(def ^:private *get-graph-salt-memoize-cache (atom {}))
|
|
|
(defn update-graph-salt-cache [graph-uuid v]
|
|
|
{:pre [(map? v)
|
|
|
@@ -1521,9 +1363,6 @@
|
|
|
val val
|
|
|
timeout' (recur))))))
|
|
|
|
|
|
-
|
|
|
-
|
|
|
-
|
|
|
(defn- assert-local-txid<=remote-txid
|
|
|
[]
|
|
|
(when-let [local-txid (last @graphs-txid)]
|
|
|
@@ -1548,7 +1387,6 @@
|
|
|
sync-state--remove-recent-remote->local-files
|
|
|
sync-state--stopped?)
|
|
|
|
|
|
-
|
|
|
(defn- filetxns=>recent-remote->local-files
|
|
|
[filetxns]
|
|
|
(let [{:keys [update-filetxns delete-filetxns rename-filetxns]}
|
|
|
@@ -1610,7 +1448,7 @@
|
|
|
(fs/unlink! repo (path/path-join repo-dir base-file) {}))
|
|
|
;; base-content != current-content, merge, do not delete
|
|
|
(p/let [merged-content (diff-merge/three-way-merge base-content "" current-content format)]
|
|
|
- (fs/write-file! repo repo-dir current-change-file merged-content {:skip-compare? true})
|
|
|
+ (fs/write-plain-text-file! repo repo-dir current-change-file merged-content {:skip-compare? true})
|
|
|
(file-handler/alter-file repo current-change-file merged-content {:re-render-root? true
|
|
|
:from-disk? true
|
|
|
:fs/event :fs/remote-file-change}))))
|
|
|
@@ -1660,7 +1498,7 @@
|
|
|
merged-content (diff-merge/three-way-merge base-content incoming-content current-content format)]
|
|
|
(when (seq merged-content)
|
|
|
(p/do!
|
|
|
- (fs/write-file! repo repo-dir current-change-file merged-content {:skip-compare? true})
|
|
|
+ (fs/write-plain-text-file! repo repo-dir current-change-file merged-content {:skip-compare? true})
|
|
|
(file-handler/alter-file repo current-change-file merged-content {:re-render-root? true
|
|
|
:from-disk? true
|
|
|
:fs/event :fs/remote-file-change})))))))
|
|
|
@@ -1687,7 +1525,7 @@
|
|
|
|
|
|
;; else
|
|
|
(p/do!
|
|
|
- (fs/write-file! repo repo-dir current-change-file merged-content {:skip-compare? true})
|
|
|
+ (fs/write-plain-text-file! repo repo-dir current-change-file merged-content {:skip-compare? true})
|
|
|
(file-handler/alter-file repo current-change-file merged-content {:re-render-root? true
|
|
|
:from-disk? true
|
|
|
:fs/event :fs/remote-file-change})))))))))))))))))
|
|
|
@@ -1724,8 +1562,8 @@
|
|
|
[recent-remote->local-file-item])
|
|
|
(<! (<delete-local-files rsapi graph-uuid base-path [relative-p*]))
|
|
|
(go (<! (timeout 5000))
|
|
|
- (swap! *sync-state sync-state--remove-recent-remote->local-files
|
|
|
- [recent-remote->local-file-item])))))
|
|
|
+ (swap! *sync-state sync-state--remove-recent-remote->local-files
|
|
|
+ [recent-remote->local-file-item])))))
|
|
|
|
|
|
(let [update-local-files-ch (if (state/enable-sync-diff-merge?)
|
|
|
(<fetch-remote-and-update-local-files graph-uuid base-path (map relative-path filetxns))
|
|
|
@@ -1781,8 +1619,8 @@
|
|
|
(not (instance? ExceptionInfo r)))]
|
|
|
;; remove these recent-remote->local-file-items 5s later
|
|
|
(go (<! (timeout 5000))
|
|
|
- (swap! *sync-state sync-state--remove-recent-remote->local-files
|
|
|
- recent-remote->local-file-items))
|
|
|
+ (swap! *sync-state sync-state--remove-recent-remote->local-files
|
|
|
+ recent-remote->local-file-items))
|
|
|
(cond
|
|
|
(instance? ExceptionInfo r) r
|
|
|
@*paused {:pause true}
|
|
|
@@ -1841,7 +1679,6 @@
|
|
|
[r]
|
|
|
(some->> (ex-cause r) str (re-find #"Request is not yet valid")))
|
|
|
|
|
|
-
|
|
|
;; type = "change" | "add" | "unlink"
|
|
|
(deftype FileChangeEvent [type dir path stat checksum]
|
|
|
IRelativePath
|
|
|
@@ -1876,7 +1713,6 @@
|
|
|
(-pr-writer [_ w _opts]
|
|
|
(write-all w (str {:type type :base-path dir :path path :size (:size stat) :checksum checksum}))))
|
|
|
|
|
|
-
|
|
|
(defn- <file-change-event=>recent-remote->local-file-item
|
|
|
"return nil when related local files not found"
|
|
|
[graph-uuid ^FileChangeEvent e]
|
|
|
@@ -1949,8 +1785,8 @@
|
|
|
(go-loop []
|
|
|
(let [{:keys [rename-event local-change]}
|
|
|
(async/alt!
|
|
|
- rename-page-event-chan ([v] {:rename-event v}) ;; {:repo X :old-path X :new-path}
|
|
|
- local-changes-chan' ([v] {:local-change v}))]
|
|
|
+ rename-page-event-chan ([v] {:rename-event v}) ;; {:repo X :old-path X :new-path}
|
|
|
+ local-changes-chan' ([v] {:local-change v}))]
|
|
|
(cond
|
|
|
rename-event
|
|
|
(let [repo-dir (config/get-repo-dir (:repo rename-event))
|
|
|
@@ -1963,13 +1799,13 @@
|
|
|
(swap! *rename-events conj k1 k2)
|
|
|
;; remove rename-events after 2s
|
|
|
(go (<! (timeout 3000))
|
|
|
- (swap! *rename-events disj k1 k2))
|
|
|
+ (swap! *rename-events disj k1 k2))
|
|
|
;; add 2 simulated file-watcher events
|
|
|
(>! ch (->FileChangeEvent "unlink" repo-dir (:old-path rename-event*) nil nil))
|
|
|
(>! ch (->FileChangeEvent "add" repo-dir (:new-path rename-event*)
|
|
|
{:mtime (tc/to-long (t/now))
|
|
|
:size 1 ; add a fake size
|
|
|
- } "fake-checksum"))
|
|
|
+ }"fake-checksum"))
|
|
|
(recur))
|
|
|
local-change
|
|
|
(cond
|
|
|
@@ -2046,7 +1882,6 @@
|
|
|
(js/localStorage.removeItem k))
|
|
|
(reset! pwd-map {}))
|
|
|
|
|
|
-
|
|
|
(defn encrypt+persist-pwd!
|
|
|
"- persist encrypted pwd at local-storage"
|
|
|
[pwd graph-uuid]
|
|
|
@@ -2094,7 +1929,6 @@
|
|
|
(when (and private-key (string/starts-with? private-key "AGE-SECRET-KEY"))
|
|
|
(set-keys¬ify graph-uuid public-key private-key)))))
|
|
|
|
|
|
-
|
|
|
(def <restored-pwd (chan (async/sliding-buffer 1)))
|
|
|
(def <restored-pwd-pub (async/pub <restored-pwd :graph-uuid))
|
|
|
|
|
|
@@ -2124,7 +1958,6 @@
|
|
|
(swap! pwd-map dissoc graph-uuid)
|
|
|
(remove-pwd! graph-uuid))
|
|
|
|
|
|
-
|
|
|
(defn- <loop-ensure-pwd&keys
|
|
|
[graph-uuid repo *stopped?]
|
|
|
(let [<restored-pwd-sub-chan (chan 1)]
|
|
|
@@ -2186,7 +2019,6 @@
|
|
|
(clear-pwd! graph-uuid)
|
|
|
(recur))))))))))
|
|
|
|
|
|
-
|
|
|
(defn- <set-env&keys
|
|
|
[prod? graph-uuid]
|
|
|
(let [{:keys [private-key public-key]} (get @pwd-map graph-uuid)]
|
|
|
@@ -2241,7 +2073,6 @@
|
|
|
(fn [_ _ _ _]
|
|
|
(offer! recent-edited-chan true)))
|
|
|
|
|
|
-
|
|
|
;;; ### sync state
|
|
|
|
|
|
(def *resume-state
|
|
|
@@ -2385,7 +2216,6 @@
|
|
|
(contains? #{::idle ::local->remote ::remote->local ::local->remote-full-sync ::remote->local-full-sync}
|
|
|
(:state sync-state)))
|
|
|
|
|
|
-
|
|
|
;;; ### remote->local syncer & local->remote syncer
|
|
|
|
|
|
(defprotocol IRemote->LocalSync
|
|
|
@@ -2641,11 +2471,11 @@
|
|
|
local-files-meta-map))))
|
|
|
|
|
|
(defrecord ^:large-vars/cleanup-todo
|
|
|
- Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state remoteapi'
|
|
|
- ^:mutable rate *txid *txid-for-get-deletion-log
|
|
|
- ^:mutable remote->local-syncer stop-chan *stopped *paused
|
|
|
+ Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state remoteapi'
|
|
|
+ ^:mutable rate *txid *txid-for-get-deletion-log
|
|
|
+ ^:mutable remote->local-syncer stop-chan *stopped *paused
|
|
|
;; control chans
|
|
|
- private-immediately-local->remote-chan private-recent-edited-chan]
|
|
|
+ private-immediately-local->remote-chan private-recent-edited-chan]
|
|
|
Object
|
|
|
(filter-file-change-events-fn [_]
|
|
|
(fn [^FileChangeEvent e]
|
|
|
@@ -2812,21 +2642,21 @@
|
|
|
<!
|
|
|
(sort-by (sort-file-metadata-fn :recent-days-range recent-10-days-range) >))
|
|
|
change-events
|
|
|
- (sequence
|
|
|
- (comp
|
|
|
+ (sequence
|
|
|
+ (comp
|
|
|
;; convert to FileChangeEvent
|
|
|
- (map #(->FileChangeEvent "change" base-path (.get-normalized-path ^FileMetadata %)
|
|
|
- {:size (:size %)} (:etag %)))
|
|
|
- (remove ignored?))
|
|
|
- diff-local-files)
|
|
|
+ (map #(->FileChangeEvent "change" base-path (.get-normalized-path ^FileMetadata %)
|
|
|
+ {:size (:size %)} (:etag %)))
|
|
|
+ (remove ignored?))
|
|
|
+ diff-local-files)
|
|
|
distinct-change-events (-> (distinct-file-change-events change-events)
|
|
|
filter-upload-files-with-reserved-chars)
|
|
|
_ (swap! *sync-state #(sync-state-reset-full-local->remote-files % distinct-change-events))
|
|
|
change-events-partitions
|
|
|
- (sequence
|
|
|
+ (sequence
|
|
|
;; partition FileChangeEvents
|
|
|
- (partition-file-change-events upload-batch-size)
|
|
|
- distinct-change-events)]
|
|
|
+ (partition-file-change-events upload-batch-size)
|
|
|
+ distinct-change-events)]
|
|
|
(println "[full-sync(local->remote)]"
|
|
|
(count (flatten change-events-partitions)) "files need to sync and"
|
|
|
(count delete-local-files) "local files need to delete")
|
|
|
@@ -2847,8 +2677,8 @@
|
|
|
[fake-recent-remote->local-file-item])
|
|
|
(<! (<delete-local-files rsapi graph-uuid base-path [(relative-path f)]))
|
|
|
(go (<! (timeout 5000))
|
|
|
- (swap! *sync-state sync-state--remove-recent-remote->local-files
|
|
|
- [fake-recent-remote->local-file-item])))))
|
|
|
+ (swap! *sync-state sync-state--remove-recent-remote->local-files
|
|
|
+ [fake-recent-remote->local-file-item])))))
|
|
|
(recur fs)))
|
|
|
|
|
|
;; 2. upload local files
|
|
|
@@ -2866,13 +2696,12 @@
|
|
|
(or need-sync-remote graph-has-been-deleted unknown stop) r)))))]
|
|
|
;; update *txid-for-get-deletion-log
|
|
|
(reset! *txid-for-get-deletion-log @*txid)
|
|
|
- r
|
|
|
- )))))))
|
|
|
+ r)))))))
|
|
|
|
|
|
;;; ### put all stuff together
|
|
|
|
|
|
(defrecord ^:large-vars/cleanup-todo
|
|
|
- SyncManager [user-uuid graph-uuid base-path *sync-state
|
|
|
+ SyncManager [user-uuid graph-uuid base-path *sync-state
|
|
|
^Local->RemoteSyncer local->remote-syncer ^Remote->LocalSyncer remote->local-syncer remoteapi'
|
|
|
^:mutable ratelimit-local-changes-chan
|
|
|
*txid *txid-for-get-deletion-log
|
|
|
@@ -3256,9 +3085,7 @@
|
|
|
local->remote-syncer (->Local->RemoteSyncer user-uuid graph-uuid
|
|
|
base-path
|
|
|
repo *sync-state remoteapi-with-stop
|
|
|
- (if (mobile-util/native-platform?)
|
|
|
- 2000
|
|
|
- 10000)
|
|
|
+ 10000
|
|
|
*txid *txid-for-get-deletion-log nil (chan) *stopped? *paused?
|
|
|
(chan 1) (chan 1))
|
|
|
remote->local-syncer (->Remote->LocalSyncer user-uuid graph-uuid base-path
|
|
|
@@ -3276,7 +3103,6 @@
|
|
|
(reset! current-sm-graph-uuid graph-uuid)
|
|
|
(sync-manager user-uuid graph-uuid base-path repo txid *sync-state)))
|
|
|
|
|
|
-
|
|
|
(defn <sync-stop []
|
|
|
(go
|
|
|
(when-let [sm ^SyncManager (state/get-file-sync-manager (state/get-current-file-sync-graph-uuid))]
|
|
|
@@ -3290,11 +3116,6 @@
|
|
|
|
|
|
(reset! current-sm-graph-uuid nil)))
|
|
|
|
|
|
-(defn <sync-local->remote-now []
|
|
|
- (go
|
|
|
- (when-let [_sm ^SyncManager (state/get-file-sync-manager (state/get-current-file-sync-graph-uuid))]
|
|
|
- (offer! immediately-local->remote-chan true))))
|
|
|
-
|
|
|
(defn sync-need-password!
|
|
|
[]
|
|
|
(when-let [sm ^SyncManager (state/get-file-sync-manager (state/get-current-file-sync-graph-uuid))]
|
|
|
@@ -3319,15 +3140,15 @@
|
|
|
(go
|
|
|
(let [r (<! (<list-remote-graphs remoteapi))
|
|
|
result
|
|
|
- (or
|
|
|
+ (or
|
|
|
;; if api call failed, assume this remote graph still exists
|
|
|
- (instance? ExceptionInfo r)
|
|
|
- (and
|
|
|
- (contains? r :Graphs)
|
|
|
- (->> (:Graphs r)
|
|
|
- (mapv :GraphUUID)
|
|
|
- set
|
|
|
- (#(contains? % local-graph-uuid)))))]
|
|
|
+ (instance? ExceptionInfo r)
|
|
|
+ (and
|
|
|
+ (contains? r :Graphs)
|
|
|
+ (->> (:Graphs r)
|
|
|
+ (mapv :GraphUUID)
|
|
|
+ set
|
|
|
+ (#(contains? % local-graph-uuid)))))]
|
|
|
|
|
|
(when-not result
|
|
|
(notification/show! (t :file-sync/graph-deleted) :warning false))
|
|
|
@@ -3347,7 +3168,6 @@
|
|
|
(when-let [graph-uuid (second @graphs-txid)]
|
|
|
(get-pwd graph-uuid)))
|
|
|
|
|
|
-
|
|
|
(defn- <connectivity-testing
|
|
|
[]
|
|
|
(go
|
|
|
@@ -3441,52 +3261,8 @@
|
|
|
(finally
|
|
|
(reset! *sync-starting false)))))))
|
|
|
|
|
|
-(defn- restart-if-stopped!
|
|
|
- [is-active?]
|
|
|
- (cond
|
|
|
- (and is-active? (graph-sync-off? (second @graphs-txid)))
|
|
|
- (<sync-start)
|
|
|
-
|
|
|
- :else
|
|
|
- (offer! pause-resume-chan is-active?)))
|
|
|
-
|
|
|
-(def app-state-changed-cursor (rum/cursor state/state :mobile/app-state-change))
|
|
|
-
|
|
|
(def finished-local->remote-chan (chan 1))
|
|
|
|
|
|
-(add-watch app-state-changed-cursor "sync"
|
|
|
- (fn [_ _ _ {:keys [is-active?]}]
|
|
|
- (cond
|
|
|
- (mobile-util/native-android?)
|
|
|
- (when-not is-active?
|
|
|
- (<sync-local->remote-now))
|
|
|
-
|
|
|
- (mobile-util/native-ios?)
|
|
|
- (let [*task-id (atom nil)]
|
|
|
- (if is-active?
|
|
|
- (restart-if-stopped! is-active?)
|
|
|
- (when (state/get-current-file-sync-graph-uuid)
|
|
|
- (p/let [task-id (.beforeExit ^js BackgroundTask
|
|
|
- (fn []
|
|
|
- (go
|
|
|
- ;; Wait for file watcher events
|
|
|
- (<! (timeout 2000))
|
|
|
- (util/drain-chan finished-local->remote-chan)
|
|
|
- (<! (<sync-local->remote-now))
|
|
|
- ;; wait at most 20s
|
|
|
- (async/alts! [finished-local->remote-chan (timeout 20000)])
|
|
|
- (p/let [active? (mobile-util/app-active?)]
|
|
|
- (when-not active?
|
|
|
- (offer! pause-resume-chan is-active?)))
|
|
|
- (<! (timeout 5000))
|
|
|
- (prn "finish task: " @*task-id)
|
|
|
- (let [opt #js {:taskId @*task-id}]
|
|
|
- (.finish ^js BackgroundTask opt)))))]
|
|
|
- (reset! *task-id task-id)))))
|
|
|
-
|
|
|
- :else
|
|
|
- nil)))
|
|
|
-
|
|
|
;;; ### some add-watches
|
|
|
|
|
|
;; TODO: replace this logic by pause/resume state
|
|
|
@@ -3517,7 +3293,6 @@
|
|
|
(<sync-start))
|
|
|
(recur))
|
|
|
|
|
|
-
|
|
|
;;; ### some sync events handler
|
|
|
|
|
|
;; re-exec remote->local-full-sync when it failed before
|
|
|
@@ -3543,6 +3318,5 @@
|
|
|
|
|
|
;;; add-tap
|
|
|
(comment
|
|
|
- (def *x (atom nil))
|
|
|
- (add-tap (fn [v] (reset! *x v)))
|
|
|
- )
|
|
|
+ (def *x (atom nil))
|
|
|
+ (add-tap (fn [v] (reset! *x v))))
|