Merge branch 'master' into fix/sync-crash

deps/graph-parser/src/logseq/graph_parser/extract.cljc

@@ -24,7 +24,7 @@
     (let [result (first (gp-util/split-last "." file-name))
           ext (string/lower-case (gp-util/get-file-ext filepath))]
       (if (or (gp-config/mldoc-support? ext) (= "edn" ext))
-        (js/decodeURIComponent (string/replace result "." "/"))
+        (gp-util/safe-decode-uri-component (string/replace result "." "/"))
         result))))
 
 (defn- get-page-name

deps/graph-parser/src/logseq/graph_parser/util.cljs

@@ -7,12 +7,18 @@
             [clojure.walk :as walk]
             [logseq.graph-parser.log :as log]))
 
+(defn safe-decode-uri-component
+  [uri]
+  (try
+    (js/decodeURIComponent uri)
+    (catch :default _
+      (log/error :decode-uri-component-failed uri)
+      uri)))
+
 (defn safe-url-decode
   [string]
   (if (string/includes? string "%")
-    (try (some-> string str (js/decodeURIComponent))
-         (catch :default _
-           string))
+    (some-> string str safe-decode-uri-component)
     string))
 
 (defn path-normalize
@@ -220,7 +226,8 @@
 ;; Source: https://github.com/logseq/logseq/blob/e7110eea6790eda5861fdedb6b02c2a78b504cd9/deps/graph-parser/src/logseq/graph_parser/extract.cljc#L35
 (defn legacy-title-parsing
   [file-name-body]
-  (js/decodeURIComponent (string/replace file-name-body "." "/")))
+  (let [title (string/replace file-name-body "." "/")]
+    (or (safe-decode-uri-component title) title)))
 
 ;; Register sanitization / parsing fns in:
 ;; logseq.graph-parser.util (parsing only)

src/electron/electron/core.cljs

@@ -77,7 +77,7 @@
                       [STATIC_URL js/__dirname])
 
            path' (.-pathname url')
-           path' (js/decodeURIComponent path')
+           path' (utils/safe-decode-uri-component path')
            path' (.join path ROOT path')]
 
        (callback #js {:path path'}))))

src/electron/electron/utils.cljs

@@ -154,3 +154,11 @@
 (defn normalize-lc
   [s]
   (normalize (string/lower-case s)))
+
+(defn safe-decode-uri-component
+  [uri]
+  (try
+    (js/decodeURIComponent uri)
+    (catch :default _
+      (println "decodeURIComponent failed: " uri)
+      uri)))

src/electron/electron/window.cljs

@@ -123,7 +123,7 @@
           new-win-handler
           (fn [e url]
             (let [url (if (string/starts-with? url "file:")
-                        (js/decodeURIComponent url) url)
+                        (utils/safe-decode-uri-component url) url)
                   url (if-not win32? (string/replace url "file://" "") url)]
               (logger/info "new-window" url)
               (if (some #(string/includes?

src/main/frontend/components/block.cljs

@@ -836,7 +836,7 @@
   [label]
   (when (and (= 1 (count label))
              (string? (last (first label))))
-    (js/decodeURIComponent (last (first label)))))
+    (gp-util/safe-decode-uri-component (last (first label)))))
 
 (defn- get-page
   [label]

src/main/frontend/components/block.css

@@ -150,6 +150,7 @@
   background-color: transparent;
   position: absolute;
   border-radius: 2px;
+  opacity: 0.6;
 }
 
 .block-children-left-border:hover {

src/main/frontend/components/file_sync.cljs

@@ -29,7 +29,8 @@
             [rum.core :as rum]
             [cljs-time.core :as t]
             [cljs-time.coerce :as tc]
-            [goog.functions :refer [debounce]]))
+            [goog.functions :refer [debounce]]
+            [logseq.graph-parser.util :as gp-util]))
 
 (declare maybe-onboarding-show)
 (declare open-icloud-graph-clone-picker)
@@ -78,7 +79,7 @@
 
      [:div.folder-tip.flex.flex-col.items-center
       [:h3
-       [:span (ui/icon "folder") [:label.pl-0.5 (js/decodeURIComponent graph-name)]]]
+       [:span (ui/icon "folder") [:label.pl-0.5 (gp-util/safe-decode-uri-component graph-name)]]]
       [:h4.px-6 (config/get-string-repo-dir repo)]
 
       (when (not (string/blank? selected-path))
@@ -439,7 +440,7 @@
 
              (map (fn [f] {:title [:div.file-item
                                    {:key (str "downloading-" f)}
-                                   (js/decodeURIComponent f)]
+                                   (gp-util/safe-decode-uri-component f)]
                            :key   (str "downloading-" f)
                            :icon  (if enabled-progress-panel?
                                     (let [progress (get sync-progress f)
@@ -458,13 +459,17 @@
                                 path (fs-sync/relative-path e)]
                             {:title [:div.file-item
                                      {:key (str "queue-" path)}
-                                     (js/decodeURIComponent path)]
+                                     (try
+                                       (gp-util/safe-decode-uri-component path)
+                                       (catch :default _
+                                         (prn "Wrong path: " path)
+                                         path))]
                              :key   (str "queue-" path)
                              :icon  (ui/icon icon)})) (take 10 queuing-files))
 
              (map (fn [f] {:title [:div.file-item
                                    {:key (str "uploading-" f)}
-                                   (js/decodeURIComponent f)]
+                                   (gp-util/safe-decode-uri-component f)]
                            :key   (str "uploading-" f)
                            :icon  (if enabled-progress-panel?
                                     (let [progress (get sync-progress f)
@@ -487,7 +492,7 @@
                                                         (if page-name
                                                           (rfe/push-state :page {:name page-name})
                                                           (rfe/push-state :file {:path full-path})))}
-                                           [:span.file-sync-item (js/decodeURIComponent (:path f))]
+                                           [:span.file-sync-item (gp-util/safe-decode-uri-component (:path f))]
                                            [:div.opacity-50 (ui/humanity-time-ago (:time f) nil)]]})))
                             (take 10 history-files)))))
 

src/main/frontend/config.cljs

@@ -359,7 +359,7 @@
                  "Local"))
          (->> (string/split repo-dir "Documents/")
               last
-              js/decodeURIComponent
+              gp-util/safe-decode-uri-component
               (str "/" (string/capitalize app-name) "/")))
     (get-repo-dir repo-dir)))
 

src/main/frontend/fs/sync.cljs

@@ -20,6 +20,7 @@
             [frontend.mobile.util :as mobile-util]
             [frontend.util :as util]
             [frontend.util.persist-var :as persist-var]
+            [frontend.util.fs :as fs-util]
             [frontend.handler.notification :as notification]
             [frontend.context.i18n :refer [t]]
             [frontend.diff :as diff]
@@ -462,22 +463,57 @@
    (contains-path? (relative-path path))
    (boolean)))
 
-(defn- diffs->partitioned-filetxns
+(defn- filter-download-files-with-reserved-chars
+  "Skip downloading file paths with reserved chars."
+  [files]
+  (let [reserved-files (filter
+                        #(fs-util/include-reserved-chars? (-relative-path %))
+                        files)]
+    (when (seq reserved-files)
+      (state/pub-event! [:ui/notify-skipped-downloading-files
+                         (map -relative-path reserved-files)])
+      (prn "Skipped downloading those file paths with reserved chars: "
+           (map -relative-path reserved-files))
+      )
+    (remove
+     #(fs-util/include-reserved-chars? (-relative-path %))
+     files)))
+
+(defn- filter-upload-files-with-reserved-chars
+  "Remove upoading file paths with reserved chars."
+  [paths]
+  (let [path-string? (string? (first paths))
+        f (if path-string?
+            fs-util/include-reserved-chars?
+            #(fs-util/include-reserved-chars? (-relative-path %)))
+        reserved-paths (filter f paths)]
+    (when (seq reserved-paths)
+      (let [paths (if path-string? reserved-paths (map -relative-path reserved-paths))]
+        (state/pub-event! [:ui/notify-files-with-reserved-chars paths])
+        (prn "Skipped uploading those file paths with reserved chars: " paths)))
+    (vec (remove f paths))))
+
+(defn- diffs->filetxns
   "transducer.
   1. diff -> `FileTxn` , see also `<get-diff`
   2. distinct redundant update type filetxns
-  3. partition filetxns, each partition contains same type filetxns,
-     for update type, at most N items in each partition
-     for delete & rename type, only 1 item in each partition.
-  4. remove update or rename filetxns if they are deleted in later filetxns.
+  3. remove update or rename filetxns if they are deleted in later filetxns.
   NOTE: this xf should apply on reversed diffs sequence (sort by txid)"
-  [n]
+  []
   (comp
    (map diff->filetxns)
    cat
    (remove ignored?)
    distinct-update-filetxns-xf
-   remove-deleted-filetxns-xf
+   remove-deleted-filetxns-xf))
+
+(defn- diffs->partitioned-filetxns
+  "partition filetxns, each partition contains same type filetxns,
+   for update type, at most N items in each partition
+   for delete & rename type, only 1 item in each partition."
+  [n]
+  (comp
+   (diffs->filetxns)
    (partition-filetxns n)))
 
 (defn- filepath+checksum->diff
@@ -2181,11 +2217,12 @@
                         {:need-remote->local-full-sync true})
 
                     (when (pos-int? latest-txid)
-                      (let [partitioned-filetxns (transduce (diffs->partitioned-filetxns download-batch-size)
+                      (let [filtered-diff-txns (-> (transduce (diffs->filetxns) conj '() (reverse diff-txns))
+                                                   filter-download-files-with-reserved-chars)
+                            partitioned-filetxns (transduce (partition-filetxns download-batch-size)
                                                             (completing (fn [r i] (conj r (reverse i)))) ;reverse
                                                             '()
-                                                            (reverse diff-txns))]
-                        ;; (swap! *sync-state #(sync-state-reset-full-remote->local-files % files))
+                                                            filtered-diff-txns)]
                         (put-sync-event! {:event :start
                                           :data  {:type       :remote->local
                                                   :graph-uuid graph-uuid
@@ -2236,11 +2273,12 @@
                                            :epoch (tc/to-epoch (t/now))}})
                   {:stop true})
               (do (println "[full-sync(remote->local)]" (count sorted-diff-remote-files) "files need to sync")
-                  (swap! *sync-state #(sync-state-reset-full-remote->local-files % sorted-diff-remote-files))
-                  (<! (.sync-files-remote->local!
-                       this (map (juxt relative-path -checksum)
-                              sorted-diff-remote-files)
-                       latest-txid))))))))))
+                  (let [filtered-files (filter-download-files-with-reserved-chars sorted-diff-remote-files)]
+                    (swap! *sync-state #(sync-state-reset-full-remote->local-files % sorted-diff-remote-files))
+                    (<! (.sync-files-remote->local!
+                         this (map (juxt relative-path -checksum)
+                                filtered-files)
+                         latest-txid)))))))))))
 
 (defn- <file-changed?
   "return true when file changed compared with remote"
@@ -2418,7 +2456,8 @@
                 _     (when (not= (count es**) (count es*))
                         (println :debug :filter-too-huge-files
                                  (mapv relative-path (set/difference (set es*) (set es**)))))
-                paths (sequence es->paths-xf es**)
+                paths (-> (sequence es->paths-xf es**)
+                          filter-upload-files-with-reserved-chars)
                 _     (println :sync-local->remote type paths)
                 r     (if (empty? paths)
                         (go @*txid)
@@ -2513,7 +2552,8 @@
                                            {:size (:size %)} (:etag %)))
                   (remove ignored?))
                  diff-local-files)
-                distinct-change-events (distinct-file-change-events change-events)
+                distinct-change-events (-> (distinct-file-change-events change-events)
+                                           filter-upload-files-with-reserved-chars)
                 _                      (swap! *sync-state #(sync-state-reset-full-local->remote-files % distinct-change-events))
                 change-events-partitions
                 (sequence

src/main/frontend/handler/events.cljs

@@ -54,6 +54,7 @@
             [frontend.handler.file-sync :as file-sync-handler]
             [frontend.components.file-sync :as file-sync]
             [frontend.components.encryption :as encryption]
+            [frontend.components.conversion :as conversion-component]
             [goog.dom :as gdom]
             [logseq.db.schema :as db-schema]
             [promesa.core :as p]
@@ -706,6 +707,56 @@
     (when (= dir (config/get-repo-dir repo))
       (fs/watch-dir! dir))))
 
+(defmethod handle :ui/notify-files-with-reserved-chars [[_ paths]]
+  (sync/<sync-stop)
+
+  (notification/show!
+   [:div
+    [:div.mb-4
+     [:div.font-semibold.mb-4.text-xl "It seems that you're using the old filename format."]
+
+     [:div
+      [:p
+       "We suggest you upgrade now to avoid some potential bugs."]
+      [:p
+       "For example, the files below have reserved characters can't be synced on some platforms."]]
+     ]
+    (ui/button
+      "Upgrade filename format"
+      :on-click (fn []
+                  (notification/clear-all!)
+                  (state/set-modal!
+                  (fn [_] (conversion-component/files-breaking-changed))
+                  {:id :filename-format-panel :center? true})))
+    [:ol.my-2
+     (for [path paths]
+       [:li path])]]
+   :warning
+   false))
+
+(defmethod handle :ui/notify-skipped-downloading-files [[_ paths]]
+  (notification/show!
+   [:div
+    [:div.mb-4
+     [:div.font-semibold.mb-4.text-xl "It seems that you're using the old filename format."]
+     [:p
+      "The files below that have reserved characters can't be saved on this device."]
+     [:div.overflow-y-auto.max-h-96
+      [:ol.my-2
+       (for [path paths]
+         [:li path])]]
+
+     [:div
+      [:p
+       "Check " [:a {:href "https://docs.logseq.com/#/page/logseq%20file%20and%20folder%20naming%20rules"
+                     :target "_blank"}
+                 "Logseq file and folder naming rules"]
+       " for more details."]
+      [:p
+       "To solve this problem, we suggest you upgrade the filename format (on Settings > Advanced > Filename format > click EDIT button) in other devices to avoid more potential bugs."]]]]
+   :warning
+   false))
+
 (defmethod handle :file/alter [[_ repo path content]]
   (p/let [_ (file-handler/alter-file repo path content {:from-disk? true})]
     (ui-handler/re-render-root!)))

src/main/frontend/handler/file_sync.cljs

@@ -14,7 +14,8 @@
             [frontend.fs :as fs]
             [cljs-time.coerce :as tc]
             [cljs-time.core :as t]
-            [frontend.storage :as storage]))
+            [frontend.storage :as storage]
+            [logseq.graph-parser.util :as gp-util]))
 
 (def *beta-unavailable? (volatile! false))
 
@@ -151,7 +152,7 @@
     (when-let [path (:file/path (db/entity file-id))]
       (let [base-path (config/get-repo-dir (state/get-current-repo))
             base-path (if (string/starts-with? base-path "file://")
-                        (js/decodeURIComponent base-path)
+                        (gp-util/safe-decode-uri-component base-path)
                         base-path)
             path*     (string/replace-first (string/replace-first path base-path "") #"^/" "")]
         (go

src/main/frontend/handler/paste.cljs

@@ -69,7 +69,7 @@
 (defn- get-whiteboard-tldr-from-text
   [text]
   (when-let [matched-text (util/safe-re-find #"<whiteboard-tldr>(.*)</whiteboard-tldr>" text)]
-    (try-parse-as-json (js/decodeURIComponent (second matched-text)))))
+    (try-parse-as-json (gp-util/safe-decode-uri-component (second matched-text)))))
 
 (defn- get-whiteboard-shape-refs-text
   [text]

src/main/frontend/mobile/intent.cljs

@@ -77,12 +77,12 @@
     (-> (string/replace template "{time}" time)
         (string/replace "{url}" (or url "")))))
 
-(defn- embed-text-file 
-  "Store external content with url into Logseq repo" 
+(defn- embed-text-file
+  "Store external content with url into Logseq repo"
   [url title]
   (p/let [time (date/get-current-time)
           title (some-> (or title (path/basename url))
-                        js/decodeURIComponent
+                        gp-util/safe-decode-uri-component
                         util/node-path.name
                         ;; make the title more user friendly
                         gp-util/page-name-sanity)
@@ -148,7 +148,7 @@
 
                       :else
                       (if (mobile-util/native-ios?)
-                        (js/decodeURIComponent v)
+                        (gp-util/safe-decode-uri-component v)
                         v))])))
 
 (defn handle-result [result]

src/main/frontend/util.cljc

@@ -506,7 +506,7 @@
    (defn safe-path-join [prefix & paths]
      (let [path (apply node-path.join (cons prefix paths))]
        (if (and (electron?) (gstring/caseInsensitiveStartsWith path "file://"))
-         (js/decodeURIComponent (subs path 7))
+         (gp-util/safe-decode-uri-component (subs path 7))
          path))))
 
 (defn trim-safe

src/main/frontend/util/fs.cljs

@@ -74,7 +74,7 @@
   (re-pattern (str "[" multiplatform-reserved-chars "]+")))
 
 (defn include-reserved-chars?
-  "Includes reserved charcters that would broken FS"
+  "Includes reserved characters that would broken FS"
   [s]
   (util/safe-re-find reserved-chars-pattern s))
 

tldraw/apps/tldraw-logseq/src/lib/shapes/LogseqPortalShape.tsx

@@ -83,7 +83,10 @@ const LogseqPortalShapeHeader = observer(
     opacity: number
     children: React.ReactNode
   }) => {
-    const bgColor = getComputedColor(fill, 'background')
+    const bgColor =
+      fill !== 'var(--ls-secondary-background-color)'
+        ? getComputedColor(fill, 'background')
+        : 'var(--ls-tertiary-background-color)'
 
     return (
       <div
@@ -95,12 +98,7 @@ const LogseqPortalShapeHeader = observer(
           className="absolute inset-0 tl-logseq-portal-header-bg"
           style={{
             opacity,
-            background:
-              type === 'P'
-                ? bgColor
-                : `linear-gradient(0deg, var(--ls-highlight-color-${
-                    fill ? fill : 'default'
-                  }), ${bgColor}`,
+            background: type === 'P' ? bgColor : `linear-gradient(0deg, transparent, ${bgColor}`,
           }}
         ></div>
         <div className="relative">{children}</div>
@@ -736,26 +734,29 @@ export class LogseqPortalShape extends TLBoxShape<LogseqPortalShapeProps> {
     }, [this.initialHeightCalculated])
 
     return (
-      <div
-        ref={cpRefContainer}
-        className="relative tl-logseq-cp-container"
-        style={{ overflow: this.props.isAutoResizing ? 'visible' : 'auto' }}
-      >
+      <>
         <div
           className="absolute inset-0 tl-logseq-cp-container-bg"
           style={{
-            background: fill
-              ? `var(--ls-highlight-color-${fill})`
-              : 'var(--ls-secondary-background-color)',
+            background:
+              fill && fill !== 'var(--ls-secondary-background-color)'
+                ? `var(--ls-highlight-color-${fill})`
+                : 'var(--ls-secondary-background-color)',
             opacity,
           }}
         ></div>
-        {this.props.blockType === 'B' && this.props.compact ? (
-          <Block blockId={pageId} />
-        ) : (
-          <Page pageName={pageId} />
-        )}
-      </div>
+        <div
+          ref={cpRefContainer}
+          className="relative tl-logseq-cp-container"
+          style={{ overflow: this.props.isAutoResizing ? 'visible' : 'auto' }}
+        >
+          {this.props.blockType === 'B' && this.props.compact ? (
+            <Block blockId={pageId} />
+          ) : (
+            <Page pageName={pageId} />
+          )}
+        </div>
+      </>
     )
   })
 
@@ -938,7 +939,7 @@ export class LogseqPortalShape extends TLBoxShape<LogseqPortalShapeProps> {
       <>
         <rect
           fill={
-            this.props.fill
+            this.props.fill && this.props.fill !== 'var(--ls-secondary-background-color)'
               ? `var(--ls-highlight-color-${this.props.fill})`
               : 'var(--ls-secondary-background-color)'
           }
@@ -953,7 +954,7 @@ export class LogseqPortalShape extends TLBoxShape<LogseqPortalShapeProps> {
         {!this.props.compact && (
           <rect
             fill={
-              this.props.fill
+              this.props.fill && this.props.fill !== 'var(--ls-secondary-background-color)'
                 ? getComputedColor(this.props.fill, 'background')
                 : 'var(--ls-tertiary-background-color)'
             }

tldraw/apps/tldraw-logseq/src/styles.css

@@ -700,7 +700,7 @@ button.tl-select-input-trigger {
 
   height: 40px;
   flex-shrink: 0;
-
+  z-index: 1;
   color: var(--ls-title-text-color);
   border-top-left-radius: 8px;
   border-top-right-radius: 8px;
@@ -926,7 +926,7 @@ html[data-theme='dark'] {
 }
 
 .tl-color-bg {
-  @apply w-full h-full rounded-sm;
+  @apply flex justify-center items-center w-full h-full rounded-sm;
 }
 
 .tl-slider-root {