|
@@ -26,7 +26,8 @@
|
|
|
[frontend.fs :as fs]
|
|
[frontend.fs :as fs]
|
|
|
[frontend.encrypt :as encrypt]
|
|
[frontend.encrypt :as encrypt]
|
|
|
[medley.core :refer [dedupe-by]]
|
|
[medley.core :refer [dedupe-by]]
|
|
|
- [rum.core :as rum]))
|
|
|
|
|
|
|
+ [rum.core :as rum]
|
|
|
|
|
+ [promesa.core :as p]))
|
|
|
|
|
|
|
|
;;; ### Commentary
|
|
;;; ### Commentary
|
|
|
;; file-sync related local files/dirs:
|
|
;; file-sync related local files/dirs:
|
|
@@ -181,8 +182,8 @@
|
|
|
[latest-txid graph-uuid user-uuid repo]
|
|
[latest-txid graph-uuid user-uuid repo]
|
|
|
{:pre [(int? latest-txid) (>= latest-txid 0)]}
|
|
{:pre [(int? latest-txid) (>= latest-txid 0)]}
|
|
|
(persist-var/-reset-value! graphs-txid [user-uuid graph-uuid latest-txid] repo)
|
|
(persist-var/-reset-value! graphs-txid [user-uuid graph-uuid latest-txid] repo)
|
|
|
- (some-> (persist-var/persist-save graphs-txid)
|
|
|
|
|
- p->c)
|
|
|
|
|
|
|
+ (p/let [_ (persist-var/persist-save graphs-txid)]
|
|
|
|
|
+ (state/pub-event! [:graph/refresh]))
|
|
|
(when (state/developer-mode?) (assert-local-txid<=remote-txid)))
|
|
(when (state/developer-mode?) (assert-local-txid<=remote-txid)))
|
|
|
|
|
|
|
|
(defn clear-graphs-txid! [repo]
|
|
(defn clear-graphs-txid! [repo]
|
|
@@ -633,7 +634,7 @@
|
|
|
(defprotocol IRSAPI
|
|
(defprotocol IRSAPI
|
|
|
(rsapi-ready? [this graph-uuid] "return true when rsapi ready")
|
|
(rsapi-ready? [this graph-uuid] "return true when rsapi ready")
|
|
|
(<key-gen [this] "generate public+private keys")
|
|
(<key-gen [this] "generate public+private keys")
|
|
|
- (<set-env [this prod? private-key public-key graph-uuid] "set environment")
|
|
|
|
|
|
|
+ (<set-env [this graph-uuid prod? private-key public-key] "set environment")
|
|
|
(<get-local-files-meta [this graph-uuid base-path filepaths] "get local files' metadata")
|
|
(<get-local-files-meta [this graph-uuid base-path filepaths] "get local files' metadata")
|
|
|
(<get-local-all-files-meta [this graph-uuid base-path] "get all local files' metadata")
|
|
(<get-local-all-files-meta [this graph-uuid base-path] "get all local files' metadata")
|
|
|
(<rename-local-file [this graph-uuid base-path from to])
|
|
(<rename-local-file [this graph-uuid base-path from to])
|
|
@@ -642,8 +643,8 @@
|
|
|
(<delete-local-files [this graph-uuid base-path filepaths])
|
|
(<delete-local-files [this graph-uuid base-path filepaths])
|
|
|
(<update-remote-files [this graph-uuid base-path filepaths local-txid] "local -> remote, return err or txid")
|
|
(<update-remote-files [this graph-uuid base-path filepaths local-txid] "local -> remote, return err or txid")
|
|
|
(<delete-remote-files [this graph-uuid base-path filepaths local-txid] "return err or txid")
|
|
(<delete-remote-files [this graph-uuid base-path filepaths local-txid] "return err or txid")
|
|
|
- (<encrypt-fnames [this fnames])
|
|
|
|
|
- (<decrypt-fnames [this fnames]))
|
|
|
|
|
|
|
+ (<encrypt-fnames [this graph-uuid fnames])
|
|
|
|
|
+ (<decrypt-fnames [this graph-uuid fnames]))
|
|
|
|
|
|
|
|
(defprotocol IRemoteAPI
|
|
(defprotocol IRemoteAPI
|
|
|
(<user-info [this] "user info")
|
|
(<user-info [this] "user info")
|
|
@@ -652,6 +653,7 @@
|
|
|
(<get-remote-graph [this graph-name-opt graph-uuid-opt] "get graph info by GRAPH-NAME-OPT or GRAPH-UUID-OPT")
|
|
(<get-remote-graph [this graph-name-opt graph-uuid-opt] "get graph info by GRAPH-NAME-OPT or GRAPH-UUID-OPT")
|
|
|
(<get-remote-file-versions [this graph-uuid filepath] "get file's version list")
|
|
(<get-remote-file-versions [this graph-uuid filepath] "get file's version list")
|
|
|
(<list-remote-graphs [this] "list all remote graphs")
|
|
(<list-remote-graphs [this] "list all remote graphs")
|
|
|
|
|
+ (<get-deletion-logs [this graph-uuid from-txid] "get deletion logs from FROM-TXID")
|
|
|
(<get-diff [this graph-uuid from-txid] "get diff from FROM-TXID, return [txns, latest-txid, min-txid]")
|
|
(<get-diff [this graph-uuid from-txid] "get diff from FROM-TXID, return [txns, latest-txid, min-txid]")
|
|
|
(<create-graph [this graph-name] "create graph")
|
|
(<create-graph [this graph-name] "create graph")
|
|
|
(<delete-graph [this graph-uuid] "delete graph")
|
|
(<delete-graph [this graph-uuid] "delete graph")
|
|
@@ -671,17 +673,17 @@
|
|
|
it happens on macos (case-insensitive fs)
|
|
it happens on macos (case-insensitive fs)
|
|
|
|
|
|
|
|
return canonicalized filepath if exists"
|
|
return canonicalized filepath if exists"
|
|
|
- [irsapi base-path filepath]
|
|
|
|
|
|
|
+ [graph-uuid irsapi base-path filepath]
|
|
|
(go
|
|
(go
|
|
|
- (let [r (<! (<get-local-files-meta irsapi "" base-path [filepath]))]
|
|
|
|
|
|
|
+ (let [r (<! (<get-local-files-meta irsapi graph-uuid base-path [filepath]))]
|
|
|
(when (some-> r first :path (not= filepath))
|
|
(when (some-> r first :path (not= filepath))
|
|
|
(-> r first :path)))))
|
|
(-> r first :path)))))
|
|
|
|
|
|
|
|
|
|
|
|
|
(defn <local-file-not-exist?
|
|
(defn <local-file-not-exist?
|
|
|
- [irsapi base-path filepath]
|
|
|
|
|
|
|
+ [graph-uuid irsapi base-path filepath]
|
|
|
(go
|
|
(go
|
|
|
- (let [r (<! (<get-local-files-meta irsapi "" base-path [filepath]))]
|
|
|
|
|
|
|
+ (let [r (<! (<get-local-files-meta irsapi graph-uuid base-path [filepath]))]
|
|
|
|
|
|
|
|
(or
|
|
(or
|
|
|
;; not found at all
|
|
;; not found at all
|
|
@@ -718,13 +720,13 @@
|
|
|
(rsapi-ready? [_ graph-uuid] (and (= graph-uuid graph-uuid') private-key' public-key'))
|
|
(rsapi-ready? [_ graph-uuid] (and (= graph-uuid graph-uuid') private-key' public-key'))
|
|
|
(<key-gen [_] (go (js->clj (<! (p->c (ipc/ipc "key-gen")))
|
|
(<key-gen [_] (go (js->clj (<! (p->c (ipc/ipc "key-gen")))
|
|
|
:keywordize-keys true)))
|
|
:keywordize-keys true)))
|
|
|
- (<set-env [_ prod? private-key public-key graph-uuid]
|
|
|
|
|
|
|
+ (<set-env [_ graph-uuid prod? private-key public-key]
|
|
|
(when (not-empty private-key)
|
|
(when (not-empty private-key)
|
|
|
(print (util/format "[%s] setting sync age-encryption passphrase..." graph-uuid)))
|
|
(print (util/format "[%s] setting sync age-encryption passphrase..." graph-uuid)))
|
|
|
(set! graph-uuid' graph-uuid)
|
|
(set! graph-uuid' graph-uuid)
|
|
|
(set! private-key' private-key)
|
|
(set! private-key' private-key)
|
|
|
(set! public-key' public-key)
|
|
(set! public-key' public-key)
|
|
|
- (p->c (ipc/ipc "set-env" (if prod? "prod" "dev") private-key public-key)))
|
|
|
|
|
|
|
+ (p->c (ipc/ipc "set-env" graph-uuid (if prod? "prod" "dev") private-key public-key)))
|
|
|
(<get-local-all-files-meta [_ graph-uuid base-path]
|
|
(<get-local-all-files-meta [_ graph-uuid base-path]
|
|
|
(go
|
|
(go
|
|
|
(let [r (<! (<retry-rsapi #(p->c (ipc/ipc "get-local-all-files-meta" graph-uuid base-path))))]
|
|
(let [r (<! (<retry-rsapi #(p->c (ipc/ipc "get-local-all-files-meta" graph-uuid base-path))))]
|
|
@@ -779,9 +781,9 @@
|
|
|
(<!
|
|
(<!
|
|
|
(<retry-rsapi
|
|
(<retry-rsapi
|
|
|
#(p->c (ipc/ipc "delete-remote-files" graph-uuid base-path filepaths local-txid token)))))))
|
|
#(p->c (ipc/ipc "delete-remote-files" graph-uuid base-path filepaths local-txid token)))))))
|
|
|
- (<encrypt-fnames [_ fnames] (go (js->clj (<! (p->c (ipc/ipc "encrypt-fnames" fnames))))))
|
|
|
|
|
- (<decrypt-fnames [_ fnames] (go
|
|
|
|
|
- (let [r (<! (p->c (ipc/ipc "decrypt-fnames" fnames)))]
|
|
|
|
|
|
|
+ (<encrypt-fnames [_ graph-uuid fnames] (go (js->clj (<! (p->c (ipc/ipc "encrypt-fnames" graph-uuid fnames))))))
|
|
|
|
|
+ (<decrypt-fnames [_ graph-uuid fnames] (go
|
|
|
|
|
+ (let [r (<! (p->c (ipc/ipc "decrypt-fnames" graph-uuid fnames)))]
|
|
|
(if (instance? ExceptionInfo r)
|
|
(if (instance? ExceptionInfo r)
|
|
|
(ex-info "decrypt-failed" {:fnames fnames} (ex-cause r))
|
|
(ex-info "decrypt-failed" {:fnames fnames} (ex-cause r))
|
|
|
(js->clj r))))))
|
|
(js->clj r))))))
|
|
@@ -804,7 +806,7 @@
|
|
|
(go (let [r (<! (p->c (.keygen mobile-util/file-sync #js {})))]
|
|
(go (let [r (<! (p->c (.keygen mobile-util/file-sync #js {})))]
|
|
|
(-> r
|
|
(-> r
|
|
|
(js->clj :keywordize-keys true)))))
|
|
(js->clj :keywordize-keys true)))))
|
|
|
- (<set-env [_ prod? secret-key public-key graph-uuid]
|
|
|
|
|
|
|
+ (<set-env [_ graph-uuid prod? secret-key public-key]
|
|
|
(set! graph-uuid' graph-uuid)
|
|
(set! graph-uuid' graph-uuid)
|
|
|
(set! private-key secret-key)
|
|
(set! private-key secret-key)
|
|
|
(set! public-key' public-key)
|
|
(set! public-key' public-key)
|
|
@@ -897,14 +899,14 @@
|
|
|
r
|
|
r
|
|
|
(get (js->clj r) "txid")))))
|
|
(get (js->clj r) "txid")))))
|
|
|
|
|
|
|
|
- (<encrypt-fnames [_ fnames]
|
|
|
|
|
|
|
+ (<encrypt-fnames [_ _graph-uuid fnames]
|
|
|
(go
|
|
(go
|
|
|
(let [r (<! (p->c (.encryptFnames mobile-util/file-sync
|
|
(let [r (<! (p->c (.encryptFnames mobile-util/file-sync
|
|
|
(clj->js {:filePaths fnames}))))]
|
|
(clj->js {:filePaths fnames}))))]
|
|
|
(if (instance? ExceptionInfo r)
|
|
(if (instance? ExceptionInfo r)
|
|
|
(.-cause r)
|
|
(.-cause r)
|
|
|
(get (js->clj r) "value")))))
|
|
(get (js->clj r) "value")))))
|
|
|
- (<decrypt-fnames [_ fnames]
|
|
|
|
|
|
|
+ (<decrypt-fnames [_ _graph-uuid fnames]
|
|
|
(go (let [r (<! (p->c (.decryptFnames mobile-util/file-sync
|
|
(go (let [r (<! (p->c (.decryptFnames mobile-util/file-sync
|
|
|
(clj->js {:filePaths fnames}))))]
|
|
(clj->js {:filePaths fnames}))))]
|
|
|
(if (instance? ExceptionInfo r)
|
|
(if (instance? ExceptionInfo r)
|
|
@@ -1103,9 +1105,9 @@
|
|
|
(recur next-continuation-token)))))))]
|
|
(recur next-continuation-token)))))))]
|
|
|
(if (instance? ExceptionInfo exp-r)
|
|
(if (instance? ExceptionInfo exp-r)
|
|
|
exp-r
|
|
exp-r
|
|
|
- (let [file-meta-list* (persistent! file-meta-list)
|
|
|
|
|
- encrypted-path-list* (persistent! encrypted-path-list)
|
|
|
|
|
- path-list-or-exp (<! (<decrypt-fnames rsapi encrypted-path-list*))]
|
|
|
|
|
|
|
+ (let [file-meta-list* (persistent! file-meta-list)
|
|
|
|
|
+ encrypted-path-list* (persistent! encrypted-path-list)
|
|
|
|
|
+ path-list-or-exp (<! (<decrypt-fnames rsapi graph-uuid encrypted-path-list*))]
|
|
|
(if (instance? ExceptionInfo path-list-or-exp)
|
|
(if (instance? ExceptionInfo path-list-or-exp)
|
|
|
path-list-or-exp
|
|
path-list-or-exp
|
|
|
(let [encrypted-path->path-map (zipmap encrypted-path-list* path-list-or-exp)]
|
|
(let [encrypted-path->path-map (zipmap encrypted-path-list* path-list-or-exp)]
|
|
@@ -1122,12 +1124,12 @@
|
|
|
(<get-remote-files-meta [this graph-uuid filepaths]
|
|
(<get-remote-files-meta [this graph-uuid filepaths]
|
|
|
{:pre [(coll? filepaths)]}
|
|
{:pre [(coll? filepaths)]}
|
|
|
(go
|
|
(go
|
|
|
- (let [encrypted-paths* (<! (<encrypt-fnames rsapi filepaths))
|
|
|
|
|
|
|
+ (let [encrypted-paths* (<! (<encrypt-fnames rsapi graph-uuid filepaths))
|
|
|
r (<! (.<request this "get_files_meta" {:GraphUUID graph-uuid :Files encrypted-paths*}))]
|
|
r (<! (.<request this "get_files_meta" {:GraphUUID graph-uuid :Files encrypted-paths*}))]
|
|
|
(if (instance? ExceptionInfo r)
|
|
(if (instance? ExceptionInfo r)
|
|
|
r
|
|
r
|
|
|
(let [encrypted-paths (mapv :FilePath r)
|
|
(let [encrypted-paths (mapv :FilePath r)
|
|
|
- paths-or-exp (<! (<decrypt-fnames rsapi encrypted-paths))]
|
|
|
|
|
|
|
+ paths-or-exp (<! (<decrypt-fnames rsapi graph-uuid encrypted-paths))]
|
|
|
(if (instance? ExceptionInfo paths-or-exp)
|
|
(if (instance? ExceptionInfo paths-or-exp)
|
|
|
paths-or-exp
|
|
paths-or-exp
|
|
|
(let [encrypted-path->path-map (zipmap encrypted-paths paths-or-exp)]
|
|
(let [encrypted-path->path-map (zipmap encrypted-paths paths-or-exp)]
|
|
@@ -1150,12 +1152,29 @@
|
|
|
|
|
|
|
|
(<get-remote-file-versions [this graph-uuid filepath]
|
|
(<get-remote-file-versions [this graph-uuid filepath]
|
|
|
(go
|
|
(go
|
|
|
- (let [encrypted-path (first (<! (<encrypt-fnames rsapi [filepath])))]
|
|
|
|
|
|
|
+ (let [encrypted-path (first (<! (<encrypt-fnames rsapi graph-uuid [filepath])))]
|
|
|
(<! (.<request this "get_file_version_list" {:GraphUUID graph-uuid :File encrypted-path})))))
|
|
(<! (.<request this "get_file_version_list" {:GraphUUID graph-uuid :File encrypted-path})))))
|
|
|
|
|
|
|
|
(<list-remote-graphs [this]
|
|
(<list-remote-graphs [this]
|
|
|
(.<request this "list_graphs"))
|
|
(.<request this "list_graphs"))
|
|
|
|
|
|
|
|
|
|
+ (<get-deletion-logs [this graph-uuid from-txid]
|
|
|
|
|
+ (go
|
|
|
|
|
+ (let [r (<! (.<request this "get_deletion_log" {:GraphUUID graph-uuid :FromTXId from-txid}))]
|
|
|
|
|
+ (if (instance? ExceptionInfo r)
|
|
|
|
|
+ r
|
|
|
|
|
+ (let [txns-with-encrypted-paths (mapv #(update % :path remove-user-graph-uuid-prefix) (:Transactions r))
|
|
|
|
|
+ encrypted-paths (mapv :path txns-with-encrypted-paths)
|
|
|
|
|
+ encrypted-path->path-map
|
|
|
|
|
+ (zipmap
|
|
|
|
|
+ encrypted-paths
|
|
|
|
|
+ (<! (<decrypt-fnames rsapi graph-uuid encrypted-paths)))
|
|
|
|
|
+ txns
|
|
|
|
|
+ (mapv
|
|
|
|
|
+ (fn [txn] (update txn :path #(get encrypted-path->path-map %)))
|
|
|
|
|
+ txns-with-encrypted-paths)]
|
|
|
|
|
+ txns)))))
|
|
|
|
|
+
|
|
|
(<get-diff [this graph-uuid from-txid]
|
|
(<get-diff [this graph-uuid from-txid]
|
|
|
;; TODO: path in transactions should be relative path(now s3 key, which includes graph-uuid and user-uuid)
|
|
;; TODO: path in transactions should be relative path(now s3 key, which includes graph-uuid and user-uuid)
|
|
|
(go
|
|
(go
|
|
@@ -1186,7 +1205,7 @@
|
|
|
encrypted-path->path-map
|
|
encrypted-path->path-map
|
|
|
(zipmap
|
|
(zipmap
|
|
|
encrypted-paths
|
|
encrypted-paths
|
|
|
- (<! (<decrypt-fnames rsapi encrypted-paths)))
|
|
|
|
|
|
|
+ (<! (<decrypt-fnames rsapi graph-uuid encrypted-paths)))
|
|
|
txns
|
|
txns
|
|
|
(mapv
|
|
(mapv
|
|
|
(fn [txn]
|
|
(fn [txn]
|
|
@@ -1315,11 +1334,11 @@
|
|
|
[@graphs-txid local-txid remote-txid])))))
|
|
[@graphs-txid local-txid remote-txid])))))
|
|
|
|
|
|
|
|
(defn- get-local-files-checksum
|
|
(defn- get-local-files-checksum
|
|
|
- [base-path relative-paths]
|
|
|
|
|
|
|
+ [graph-uuid base-path relative-paths]
|
|
|
(go
|
|
(go
|
|
|
(into {}
|
|
(into {}
|
|
|
(map (juxt #(.-path ^FileMetadata %) #(.-etag ^FileMetadata %)))
|
|
(map (juxt #(.-path ^FileMetadata %) #(.-etag ^FileMetadata %)))
|
|
|
- (<! (<get-local-files-meta rsapi "" base-path relative-paths)))))
|
|
|
|
|
|
|
+ (<! (<get-local-files-meta rsapi graph-uuid base-path relative-paths)))))
|
|
|
|
|
|
|
|
(declare sync-state--add-current-local->remote-files
|
|
(declare sync-state--add-current-local->remote-files
|
|
|
sync-state--add-current-remote->local-files
|
|
sync-state--add-current-remote->local-files
|
|
@@ -1387,7 +1406,8 @@
|
|
|
(remove nil?))]
|
|
(remove nil?))]
|
|
|
|
|
|
|
|
(doseq [relative-p (map relative-path filetxns)]
|
|
(doseq [relative-p (map relative-path filetxns)]
|
|
|
- (when-some [relative-p* (<! (<case-different-local-file-exist? rsapi base-path relative-p))]
|
|
|
|
|
|
|
+ (when-some [relative-p*
|
|
|
|
|
+ (<! (<case-different-local-file-exist? graph-uuid rsapi base-path relative-p))]
|
|
|
(let [recent-remote->local-file-item {:remote->local-type :delete
|
|
(let [recent-remote->local-file-item {:remote->local-type :delete
|
|
|
:checksum nil
|
|
:checksum nil
|
|
|
:path relative-p*}]
|
|
:path relative-p*}]
|
|
@@ -1414,7 +1434,7 @@
|
|
|
(.-deleted? (first filetxns))
|
|
(.-deleted? (first filetxns))
|
|
|
(let [filetxn (first filetxns)]
|
|
(let [filetxn (first filetxns)]
|
|
|
(assert (= 1 (count filetxns)))
|
|
(assert (= 1 (count filetxns)))
|
|
|
- (if (<! (<local-file-not-exist? rsapi base-path (relative-path filetxn)))
|
|
|
|
|
|
|
+ (if (<! (<local-file-not-exist? graph-uuid rsapi base-path (relative-path filetxn)))
|
|
|
;; not exist, ignore
|
|
;; not exist, ignore
|
|
|
true
|
|
true
|
|
|
(let [r (<! (<delete-local-files rsapi graph-uuid base-path [(relative-path filetxn)]))]
|
|
(let [r (<! (<delete-local-files rsapi graph-uuid base-path [(relative-path filetxn)]))]
|
|
@@ -1528,7 +1548,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
(defn- <file-change-event=>recent-remote->local-file-item
|
|
(defn- <file-change-event=>recent-remote->local-file-item
|
|
|
- [^FileChangeEvent e]
|
|
|
|
|
|
|
+ [graph-uuid ^FileChangeEvent e]
|
|
|
(go
|
|
(go
|
|
|
(let [tp (case (.-type e)
|
|
(let [tp (case (.-type e)
|
|
|
("add" "change") :update
|
|
("add" "change") :update
|
|
@@ -1536,7 +1556,7 @@
|
|
|
path (relative-path e)]
|
|
path (relative-path e)]
|
|
|
{:remote->local-type tp
|
|
{:remote->local-type tp
|
|
|
:checksum (if (= tp :delete) nil
|
|
:checksum (if (= tp :delete) nil
|
|
|
- (val (first (<! (get-local-files-checksum (.-dir e) [path])))))
|
|
|
|
|
|
|
+ (val (first (<! (get-local-files-checksum graph-uuid (.-dir e) [path])))))
|
|
|
:path path})))
|
|
:path path})))
|
|
|
|
|
|
|
|
(defn- distinct-file-change-events-xf
|
|
(defn- distinct-file-change-events-xf
|
|
@@ -1571,6 +1591,7 @@
|
|
|
(map #(partition-all n %))
|
|
(map #(partition-all n %))
|
|
|
cat))
|
|
cat))
|
|
|
|
|
|
|
|
|
|
+(declare sync-state--valid-to-accept-filewatcher-event?)
|
|
|
(defonce local-changes-chan (chan (async/dropping-buffer 1000)))
|
|
(defonce local-changes-chan (chan (async/dropping-buffer 1000)))
|
|
|
(defn file-watch-handler
|
|
(defn file-watch-handler
|
|
|
"file-watcher callback"
|
|
"file-watcher callback"
|
|
@@ -1578,12 +1599,13 @@
|
|
|
(when-let [current-graph (state/get-current-repo)]
|
|
(when-let [current-graph (state/get-current-repo)]
|
|
|
(when (string/ends-with? current-graph dir)
|
|
(when (string/ends-with? current-graph dir)
|
|
|
(let [sync-state (state/get-file-sync-state current-graph)]
|
|
(let [sync-state (state/get-file-sync-state current-graph)]
|
|
|
- (when (and sync-state (not (sync-state--stopped? sync-state)))
|
|
|
|
|
|
|
+ (when (and sync-state (sync-state--valid-to-accept-filewatcher-event? sync-state))
|
|
|
(when (or (:mtime stat) (= type "unlink"))
|
|
(when (or (:mtime stat) (= type "unlink"))
|
|
|
(go
|
|
(go
|
|
|
(let [path (remove-dir-prefix dir path)
|
|
(let [path (remove-dir-prefix dir path)
|
|
|
files-meta (and (not= "unlink" type)
|
|
files-meta (and (not= "unlink" type)
|
|
|
- (<! (<get-local-files-meta rsapi "" dir [path])))
|
|
|
|
|
|
|
+ (<! (<get-local-files-meta
|
|
|
|
|
+ rsapi (:current-syncing-graph-uuid sync-state) dir [path])))
|
|
|
checksum (and (coll? files-meta) (some-> files-meta first :etag))]
|
|
checksum (and (coll? files-meta) (some-> files-meta first :etag))]
|
|
|
(>! local-changes-chan (->FileChangeEvent type dir path stat checksum))))))))))
|
|
(>! local-changes-chan (->FileChangeEvent type dir path stat checksum))))))))))
|
|
|
|
|
|
|
@@ -1834,7 +1856,7 @@
|
|
|
(let [{:keys [private-key public-key]} (get @pwd-map graph-uuid)]
|
|
(let [{:keys [private-key public-key]} (get @pwd-map graph-uuid)]
|
|
|
(assert (and private-key public-key) (pr-str :private-key private-key :public-key public-key
|
|
(assert (and private-key public-key) (pr-str :private-key private-key :public-key public-key
|
|
|
:pwd-map @pwd-map))
|
|
:pwd-map @pwd-map))
|
|
|
- (<set-env rsapi prod? private-key public-key graph-uuid)))
|
|
|
|
|
|
|
+ (<set-env rsapi graph-uuid prod? private-key public-key)))
|
|
|
|
|
|
|
|
(defn- <ensure-set-env&keys
|
|
(defn- <ensure-set-env&keys
|
|
|
[graph-uuid *stopped?]
|
|
[graph-uuid *stopped?]
|
|
@@ -2018,6 +2040,13 @@
|
|
|
{:pre [(s/valid? ::sync-state sync-state)]}
|
|
{:pre [(s/valid? ::sync-state sync-state)]}
|
|
|
(= ::stop (:state sync-state)))
|
|
(= ::stop (:state sync-state)))
|
|
|
|
|
|
|
|
|
|
+(defn sync-state--valid-to-accept-filewatcher-event?
|
|
|
|
|
+ [sync-state]
|
|
|
|
|
+ {:pre [(s/valid? ::sync-state sync-state)]}
|
|
|
|
|
+ (contains? #{::idle ::local->remote ::remote->local ::local->remote-full-sync ::remote->local-full-sync}
|
|
|
|
|
+ (:state sync-state)))
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
;;; ### remote->local syncer & local->remote syncer
|
|
;;; ### remote->local syncer & local->remote syncer
|
|
|
|
|
|
|
|
(defprotocol IRemote->LocalSync
|
|
(defprotocol IRemote->LocalSync
|
|
@@ -2140,18 +2169,18 @@
|
|
|
(case (.-type e)
|
|
(case (.-type e)
|
|
|
"unlink"
|
|
"unlink"
|
|
|
;; keep this e when it's not found
|
|
;; keep this e when it's not found
|
|
|
- (<! (<local-file-not-exist? rsapi basepath r-path))
|
|
|
|
|
|
|
+ (<! (<local-file-not-exist? graph-uuid rsapi basepath r-path))
|
|
|
|
|
|
|
|
("add" "change")
|
|
("add" "change")
|
|
|
;; 1. local file exists
|
|
;; 1. local file exists
|
|
|
;; 2. compare with remote file, and changed
|
|
;; 2. compare with remote file, and changed
|
|
|
- (and (not (<! (<local-file-not-exist? rsapi basepath r-path)))
|
|
|
|
|
|
|
+ (and (not (<! (<local-file-not-exist? graph-uuid rsapi basepath r-path)))
|
|
|
(<! (<file-changed? graph-uuid r-path basepath)))))))
|
|
(<! (<file-changed? graph-uuid r-path basepath)))))))
|
|
|
|
|
|
|
|
(defn- <filter-checksum-not-consistent
|
|
(defn- <filter-checksum-not-consistent
|
|
|
"filter out FileChangeEvents checksum changed,
|
|
"filter out FileChangeEvents checksum changed,
|
|
|
compare checksum in FileChangeEvent and checksum calculated now"
|
|
compare checksum in FileChangeEvent and checksum calculated now"
|
|
|
- [es]
|
|
|
|
|
|
|
+ [graph-uuid es]
|
|
|
{:pre [(or (nil? es) (coll? es))
|
|
{:pre [(or (nil? es) (coll? es))
|
|
|
(every? #(instance? FileChangeEvent %) es)]}
|
|
(every? #(instance? FileChangeEvent %) es)]}
|
|
|
(go
|
|
(go
|
|
@@ -2159,7 +2188,8 @@
|
|
|
(if (= "unlink" (.-type ^FileChangeEvent (first es)))
|
|
(if (= "unlink" (.-type ^FileChangeEvent (first es)))
|
|
|
es
|
|
es
|
|
|
(let [base-path (.-dir (first es))
|
|
(let [base-path (.-dir (first es))
|
|
|
- files-meta (<! (<get-local-files-meta rsapi "" base-path (mapv relative-path es)))
|
|
|
|
|
|
|
+ files-meta (<! (<get-local-files-meta
|
|
|
|
|
+ rsapi graph-uuid base-path (mapv relative-path es)))
|
|
|
current-checksum-map (when (coll? files-meta) (into {} (mapv (juxt :path :etag) files-meta)))
|
|
current-checksum-map (when (coll? files-meta) (into {} (mapv (juxt :path :etag) files-meta)))
|
|
|
origin-checksum-map (into {} (mapv (juxt relative-path #(.-checksum ^FileChangeEvent %)) es))
|
|
origin-checksum-map (into {} (mapv (juxt relative-path #(.-checksum ^FileChangeEvent %)) es))
|
|
|
origin-map (into {} (mapv (juxt relative-path identity) es))]
|
|
origin-map (into {} (mapv (juxt relative-path identity) es))]
|
|
@@ -2189,6 +2219,21 @@
|
|
|
(every? #(instance? FileChangeEvent %) es)]}
|
|
(every? #(instance? FileChangeEvent %) es)]}
|
|
|
(filterv filter-too-huge-files-aux es))
|
|
(filterv filter-too-huge-files-aux es))
|
|
|
|
|
|
|
|
|
|
+(defn- filter-local-files-in-deletion-logs
|
|
|
|
|
+ [local-all-files-meta deletion-logs]
|
|
|
|
|
+ (let [deletion-logs-map (into {} (map (juxt :path identity)) deletion-logs)
|
|
|
|
|
+ *keep (transient #{})
|
|
|
|
|
+ *delete (transient #{})]
|
|
|
|
|
+ (doseq [f local-all-files-meta]
|
|
|
|
|
+ (let [epoch-long (some-> (get deletion-logs-map (:path f))
|
|
|
|
|
+ :epoch
|
|
|
|
|
+ (* 1000))]
|
|
|
|
|
+ (if (and epoch-long (> epoch-long (:last-modified f)))
|
|
|
|
|
+ (conj! *delete f)
|
|
|
|
|
+ (conj! *keep f))))
|
|
|
|
|
+ {:keep (persistent! *keep)
|
|
|
|
|
+ :delete (persistent! *delete)}))
|
|
|
|
|
+
|
|
|
|
|
|
|
|
(defrecord ^:large-vars/cleanup-todo
|
|
(defrecord ^:large-vars/cleanup-todo
|
|
|
Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state remoteapi
|
|
Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state remoteapi
|
|
@@ -2206,10 +2251,11 @@
|
|
|
true)
|
|
true)
|
|
|
(or (string/starts-with? (.-dir e) base-path)
|
|
(or (string/starts-with? (.-dir e) base-path)
|
|
|
(string/starts-with? (str "file://" (.-dir e)) base-path)) ; valid path prefix
|
|
(string/starts-with? (str "file://" (.-dir e)) base-path)) ; valid path prefix
|
|
|
- (not (ignored? e)) ;not ignored
|
|
|
|
|
|
|
+ (not (ignored? e)) ;not ignored
|
|
|
;; download files will also trigger file-change-events, ignore them
|
|
;; download files will also trigger file-change-events, ignore them
|
|
|
(let [r (not (contains? (:recent-remote->local-files @*sync-state)
|
|
(let [r (not (contains? (:recent-remote->local-files @*sync-state)
|
|
|
- (<! (<file-change-event=>recent-remote->local-file-item e))))]
|
|
|
|
|
|
|
+ (<! (<file-change-event=>recent-remote->local-file-item
|
|
|
|
|
+ graph-uuid e))))]
|
|
|
(when (and (true? r)
|
|
(when (and (true? r)
|
|
|
(seq (:recent-remote->local-files @*sync-state)))
|
|
(seq (:recent-remote->local-files @*sync-state)))
|
|
|
(println :debug (:recent-remote->local-files @*sync-state) e))
|
|
(println :debug (:recent-remote->local-files @*sync-state) e))
|
|
@@ -2252,12 +2298,12 @@
|
|
|
(<sync-local->remote! [_ es]
|
|
(<sync-local->remote! [_ es]
|
|
|
(if (empty? es)
|
|
(if (empty? es)
|
|
|
(go {:succ true})
|
|
(go {:succ true})
|
|
|
- (let [type (.-type ^FileChangeEvent (first es))
|
|
|
|
|
- es->paths-xf (comp
|
|
|
|
|
- (map #(relative-path %))
|
|
|
|
|
- (remove ignored?))]
|
|
|
|
|
|
|
+ (let [type (.-type ^FileChangeEvent (first es))
|
|
|
|
|
+ es->paths-xf (comp
|
|
|
|
|
+ (map #(relative-path %))
|
|
|
|
|
+ (remove ignored?))]
|
|
|
(go
|
|
(go
|
|
|
- (let [es* (<! (<filter-checksum-not-consistent es))
|
|
|
|
|
|
|
+ (let [es* (<! (<filter-checksum-not-consistent graph-uuid es))
|
|
|
_ (when (not= (count es*) (count es))
|
|
_ (when (not= (count es*) (count es))
|
|
|
(println :debug :filter-checksum-changed
|
|
(println :debug :filter-checksum-changed
|
|
|
(mapv relative-path (set/difference (set es) (set es*)))))
|
|
(mapv relative-path (set/difference (set es) (set es*)))))
|
|
@@ -2315,7 +2361,9 @@
|
|
|
(go
|
|
(go
|
|
|
(let [remote-all-files-meta-c (<get-remote-all-files-meta remoteapi graph-uuid)
|
|
(let [remote-all-files-meta-c (<get-remote-all-files-meta remoteapi graph-uuid)
|
|
|
local-all-files-meta-c (<get-local-all-files-meta rsapi graph-uuid base-path)
|
|
local-all-files-meta-c (<get-local-all-files-meta rsapi graph-uuid base-path)
|
|
|
- remote-all-files-meta-or-exp (<! remote-all-files-meta-c)]
|
|
|
|
|
|
|
+ deletion-logs-c (<get-deletion-logs remoteapi graph-uuid @*txid)
|
|
|
|
|
+ remote-all-files-meta-or-exp (<! remote-all-files-meta-c)
|
|
|
|
|
+ deletion-logs (<! deletion-logs-c)]
|
|
|
(if (or (storage-exceed-limit? remote-all-files-meta-or-exp)
|
|
(if (or (storage-exceed-limit? remote-all-files-meta-or-exp)
|
|
|
(sync-stop-when-api-flying? remote-all-files-meta-or-exp)
|
|
(sync-stop-when-api-flying? remote-all-files-meta-or-exp)
|
|
|
(decrypt-exp? remote-all-files-meta-or-exp))
|
|
(decrypt-exp? remote-all-files-meta-or-exp))
|
|
@@ -2326,6 +2374,8 @@
|
|
|
{:stop true})
|
|
{:stop true})
|
|
|
(let [remote-all-files-meta remote-all-files-meta-or-exp
|
|
(let [remote-all-files-meta remote-all-files-meta-or-exp
|
|
|
local-all-files-meta (<! local-all-files-meta-c)
|
|
local-all-files-meta (<! local-all-files-meta-c)
|
|
|
|
|
+ {local-all-files-meta :keep delete-local-files :delete}
|
|
|
|
|
+ (filter-local-files-in-deletion-logs local-all-files-meta deletion-logs)
|
|
|
diff-local-files (diff-file-metadata-sets local-all-files-meta remote-all-files-meta)
|
|
diff-local-files (diff-file-metadata-sets local-all-files-meta remote-all-files-meta)
|
|
|
change-events
|
|
change-events
|
|
|
(sequence
|
|
(sequence
|
|
@@ -2341,7 +2391,25 @@
|
|
|
(partition-file-change-events 10)
|
|
(partition-file-change-events 10)
|
|
|
(distinct-file-change-events change-events))]
|
|
(distinct-file-change-events change-events))]
|
|
|
(println "[full-sync(local->remote)]"
|
|
(println "[full-sync(local->remote)]"
|
|
|
- (count (flatten change-events-partitions)) "files need to sync")
|
|
|
|
|
|
|
+ (count (flatten change-events-partitions)) "files need to sync and"
|
|
|
|
|
+ (count delete-local-files) "local files need to delete")
|
|
|
|
|
+ ;; 1. delete local files
|
|
|
|
|
+ (loop [[f & fs] delete-local-files]
|
|
|
|
|
+ (when f
|
|
|
|
|
+ (let [relative-p (relative-path f)]
|
|
|
|
|
+ (when-not (<! (<local-file-not-exist? graph-uuid rsapi base-path relative-p))
|
|
|
|
|
+ (let [fake-recent-remote->local-file-item {:remote->local-type :delete
|
|
|
|
|
+ :checksum nil
|
|
|
|
|
+ :path relative-p}]
|
|
|
|
|
+ (swap! *sync-state sync-state--add-recent-remote->local-files
|
|
|
|
|
+ [fake-recent-remote->local-file-item])
|
|
|
|
|
+ (<! (<delete-local-files rsapi graph-uuid base-path [(relative-path f)]))
|
|
|
|
|
+ (go (<! (timeout 5000))
|
|
|
|
|
+ (swap! *sync-state sync-state--remove-recent-remote->local-files
|
|
|
|
|
+ [fake-recent-remote->local-file-item])))))
|
|
|
|
|
+ (recur fs)))
|
|
|
|
|
+
|
|
|
|
|
+ ;; 2. upload local files
|
|
|
(loop [es-partitions change-events-partitions]
|
|
(loop [es-partitions change-events-partitions]
|
|
|
(if @*stopped
|
|
(if @*stopped
|
|
|
{:stop true}
|
|
{:stop true}
|
|
@@ -2779,7 +2847,6 @@
|
|
|
(go
|
|
(go
|
|
|
(when (and (graph-sync-off? repo) @network-online-cursor)
|
|
(when (and (graph-sync-off? repo) @network-online-cursor)
|
|
|
(<! (p->c (persist-var/-load graphs-txid)))
|
|
(<! (p->c (persist-var/-load graphs-txid)))
|
|
|
-
|
|
|
|
|
(let [[user-uuid graph-uuid txid] @graphs-txid]
|
|
(let [[user-uuid graph-uuid txid] @graphs-txid]
|
|
|
(when (and user-uuid graph-uuid txid
|
|
(when (and user-uuid graph-uuid txid
|
|
|
(user/logged-in?)
|
|
(user/logged-in?)
|