enhance(sync): filter pages whose page-name is too long

src/main/frontend/fs/sync.cljs

@@ -1177,12 +1177,14 @@
               paths-or-exp
               (let [encrypted-path->path-map (zipmap encrypted-paths paths-or-exp)]
                 (into #{}
-                      (map #(->FileMetadata (:Size %)
-                                            (:Checksum %)
-                                            (get encrypted-path->path-map (:FilePath %))
-                                            (:FilePath %)
-                                            (:LastModified %)
-                                            true nil))
+                      (comp
+                       (filter #(not= "filepath too long" (:Error %)))
+                       (map #(->FileMetadata (:Size %)
+                                             (:Checksum %)
+                                             (get encrypted-path->path-map (:FilePath %))
+                                             (:FilePath %)
+                                             (:LastModified %)
+                                             true nil)))
                       r))))))))
 
   (<get-remote-graph [this graph-name-opt graph-uuid-opt]
@@ -2326,25 +2328,40 @@
     {:keep   (persistent! *keep)
      :delete (persistent! *delete)}))
 
+(defn- <filter-too-long-filename
+  [graph-uuid local-files-meta]
+  (go (let [origin-fnames    (mapv :path local-files-meta)
+            encrypted-fnames (<! (<encrypt-fnames rsapi graph-uuid origin-fnames))
+            fnames-map (zipmap origin-fnames encrypted-fnames)
+            local-files-meta-map (into {} (map (fn [meta] [(:path meta) meta])) local-files-meta)]
+        (sequence
+         (comp
+          (filter
+           (fn [[path _]]
+                                        ; 951 = (- 1024 36 36 1)
+                                        ; 1024 - user-uuid - graph-uuid - '/'
+             (<= (count (get fnames-map path)) 951)))
+          (map second))
+         local-files-meta-map))))
 
 (defrecord ^:large-vars/cleanup-todo
-  Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state remoteapi
-                       ^:mutable rate *txid ^:mutable remote->local-syncer stop-chan *stopped *paused
-                       ;; control chans
-                       private-immediately-local->remote-chan private-recent-edited-chan]
+ Local->RemoteSyncer [user-uuid graph-uuid base-path repo *sync-state remoteapi
+                      ^:mutable rate *txid ^:mutable remote->local-syncer stop-chan *stopped *paused
+                         ;; control chans
+                      private-immediately-local->remote-chan private-recent-edited-chan]
   Object
   (filter-file-change-events-fn [_]
     (fn [^FileChangeEvent e]
       (go (and (instance? FileChangeEvent e)
                (if-let [mtime (:mtime (.-stat e))]
-                 ;; if mtime is not nil, it should be after (- now 1min)
-                 ;; ignore events too early
+                   ;; if mtime is not nil, it should be after (- now 1min)
+                   ;; ignore events too early
                  (> (* 1000 mtime) (tc/to-long (t/minus (t/now) (t/minutes 1))))
                  true)
                (or (string/starts-with? (.-dir e) base-path)
                    (string/starts-with? (str "file://" (.-dir e)) base-path)) ; valid path prefix
                (not (ignored? e))     ;not ignored
-               ;; download files will also trigger file-change-events, ignore them
+                 ;; download files will also trigger file-change-events, ignore them
                (not (contains? (:recent-remote->local-files @*sync-state)
                                (<! (<file-change-event=>recent-remote->local-file-item
                                     graph-uuid e))))))))
@@ -2419,7 +2436,7 @@
                   {:need-sync-remote true})
 
               (need-reset-local-txid? r*) ;; TODO: this cond shouldn't be true,
-              ;; but some potential bugs cause local-txid > remote-txid
+                ;; but some potential bugs cause local-txid > remote-txid
               (let [remote-graph-info-or-ex (<! (<get-remote-graph remoteapi nil graph-uuid))
                     remote-txid             (:TXId remote-graph-info-or-ex)]
                 (if (or (instance? ExceptionInfo remote-graph-info-or-ex) (nil? remote-txid))
@@ -2442,7 +2459,7 @@
               succ?                   ; succ
               (do
                 (println "sync-local->remote! update txid" r*)
-                ;; persist txid
+                  ;; persist txid
                 (<! (<update-graphs-txid! r* graph-uuid user-uuid repo))
                 (reset! *txid r*)
                 {:succ true})
@@ -2483,11 +2500,13 @@
                 (filter-local-files-in-deletion-logs local-all-files-meta deletion-logs-or-exp)
                 recent-10-days-range   ((juxt #(tc/to-long (t/minus % (t/days 10))) #(tc/to-long %)) (t/today))
                 diff-local-files       (->> (diff-file-metadata-sets local-all-files-meta remote-all-files-meta)
+                                            (<filter-too-long-filename graph-uuid)
+                                            <!
                                             (sort-by (sort-file-metadata-fn :recent-days-range recent-10-days-range) >))
                 change-events
                 (sequence
                  (comp
-                  ;; convert to FileChangeEvent
+                    ;; convert to FileChangeEvent
                   (map #(->FileChangeEvent "change" base-path (.get-normalized-path ^FileMetadata %)
                                            {:size (:size %)} (:etag %)))
                   (remove ignored?))
@@ -2496,7 +2515,7 @@
                 _                      (swap! *sync-state #(sync-state-reset-full-local->remote-files % distinct-change-events))
                 change-events-partitions
                 (sequence
-                 ;; partition FileChangeEvents
+                   ;; partition FileChangeEvents
                  (partition-file-change-events upload-batch-size)
                  distinct-change-events)]
             (println "[full-sync(local->remote)]"
@@ -2507,7 +2526,7 @@
                                       :graph-uuid graph-uuid
                                       :full-sync? true
                                       :epoch      (tc/to-epoch (t/now))}})
-            ;; 1. delete local files
+              ;; 1. delete local files
             (loop [[f & fs] delete-local-files]
               (when f
                 (let [relative-p (relative-path f)]
@@ -2523,7 +2542,7 @@
                                  [fake-recent-remote->local-file-item])))))
                 (recur fs)))
 
-            ;; 2. upload local files
+              ;; 2. upload local files
             (loop [es-partitions change-events-partitions]
               (if @*stopped
                 {:stop true}