Refactor path handling on mobile platforms (#6402)

* refactor(mobile): use unescaped uri across app
* refactor(android): use uri for all path

android/app/src/main/java/com/logseq/app/FolderPicker.java

@@ -63,7 +63,7 @@ public class FolderPicker extends Plugin {
         if (path == null || path.isEmpty()) {
             call.reject("Cannot support this directory type: " + docUri);
         } else {
-            ret.put("path", path);
+            ret.put("path", "file://" + path);
             call.resolve(ret);
         }
     }

android/app/src/main/java/com/logseq/app/FsWatcher.java

@@ -133,7 +133,7 @@ public class FsWatcher extends Plugin {
         // path.
         File f = new File(path);
         obj.put("path", Uri.fromFile(f));
-        obj.put("dir", mPath);
+        obj.put("dir", "file://" + mPath);
 
         switch (event) {
             case FileObserver.CLOSE_WRITE:

src/main/frontend/config.cljs

@@ -305,11 +305,15 @@
     path
     (util/node-path.join (get-repo-dir repo-url) path)))
 
+;; FIXME: There is another get-file-path at src/main/frontend/fs/capacitor_fs.cljs
 (defn get-file-path
   "Normalization happens here"
   [repo-url relative-path]
   (when (and repo-url relative-path)
     (let [path (cond
+                 (demo-graph?)
+                 nil
+
                  (and (util/electron?) (local-db? repo-url))
                  (let [dir (get-repo-dir repo-url)]
                    (if (string/starts-with? relative-path dir)
@@ -319,7 +323,7 @@
 
                  (and (mobile-util/native-ios?) (local-db? repo-url))
                  (let [dir (get-repo-dir repo-url)]
-                   (js/decodeURI (str dir relative-path)))
+                   (str dir relative-path))
 
                  (and (mobile-util/native-android?) (local-db? repo-url))
                  (let [dir (get-repo-dir repo-url)
@@ -334,7 +338,7 @@
 
                  :else
                  relative-path)]
-      (gp-util/path-normalize path))))
+      (and (not-empty path) (gp-util/path-normalize path)))))
 
 (defn get-config-path
   ([]

src/main/frontend/fs/capacitor_fs.cljs

@@ -18,27 +18,58 @@
     []
     (.ensureDocuments mobile-util/ios-file-container)))
 
-(defn check-permission-android []
-  (p/let [permission (.checkPermissions Filesystem)
-          permission (-> permission
-                         bean/->clj
-                         :publicStorage)]
-    (when-not (= permission "granted")
-      (p/do!
-       (.requestPermissions Filesystem)))))
+(when (mobile-util/native-android?)
+  (defn- android-check-permission []
+    (p/let [permission (.checkPermissions Filesystem)
+            permission (-> permission
+                           bean/->clj
+                           :publicStorage)]
+      (when-not (= permission "granted")
+        (p/do!
+         (.requestPermissions Filesystem))))))
 
-(defn- clean-uri
-  [uri]
-  (when (string? uri)
-    (util/url-decode uri)))
+(defn- <write-file-with-utf8
+  [path content]
+  (when-not (string/blank? path)
+    (-> (p/chain (.writeFile Filesystem (clj->js {:path path
+                                                  :data content
+                                                  :encoding (.-UTF8 Encoding)
+                                                  :recursive true}))
+                 #(js->clj % :keywordize-keys true))
+        (p/catch (fn [error]
+                   (js/console.error "writeFile Error: " path ": " error)
+                   nil)))))
 
-(defn- read-file-utf8
+(defn- <read-file-with-utf8
   [path]
   (when-not (string/blank? path)
-    (.readFile Filesystem
-               (clj->js
-                {:path path
-                 :encoding (.-UTF8 Encoding)}))))
+    (-> (p/chain (.readFile Filesystem (clj->js {:path path
+                                                 :encoding (.-UTF8 Encoding)}))
+                 #(js->clj % :keywordize-keys true)
+                 #(get % :data nil))
+        (p/catch (fn [error]
+                   (js/console.error "readFile Error: " path ": " error)
+                   nil)))))
+
+(defn- <readdir [path]
+  (-> (p/chain (.readdir Filesystem (clj->js {:path path}))
+               js->clj
+               #(get % "files" nil))
+      (p/catch (fn [error]
+                 (js/console.error "readdir Error: " path ": " error)
+                 nil))))
+
+(defn- <stat [path]
+  (-> (p/chain (.stat Filesystem (clj->js {:path path}))
+               #(js->clj % :keywordize-keys true)
+               #(update % :type (fn [v]
+                                  (case v
+                                    "NSFileTypeDirectory" "directory"
+                                    "NSFileTypeRegular" "file"
+                                    v))))
+      (p/catch (fn [error]
+                 (js/console.error "stat Error: " path ": " error)
+                 nil))))
 
 (defn readdir
   "readdir recursively"
@@ -48,10 +79,7 @@
                    (if (empty? dirs)
                      result
                      (p/let [d (first dirs)
-                             files (.readdir Filesystem (clj->js {:path d}))
-                             files (-> files
-                                       js->clj
-                                       (get "files" []))
+                             files (<readdir d)
                              files (->> files
                                         (remove (fn [file]
                                                   (or (string/starts-with? file ".")
@@ -61,44 +89,31 @@
                                                       (= file "bak")))))
                              files (->> files
                                         (map (fn [file]
+                                               ;; TODO: use uri-join
                                                (str (string/replace d #"/+$" "")
                                                     "/"
                                                     (if (mobile-util/native-ios?)
-                                                      (util/url-encode file)
+                                                      (js/encodeURI file)
                                                       file)))))
-                             files-with-stats (p/all
-                                               (mapv
-                                                (fn [file]
-                                                  (p/chain
-                                                   (.stat Filesystem (clj->js {:path file}))
-                                                   #(js->clj % :keywordize-keys true)))
-                                                files))
+                             files-with-stats (p/all (mapv <stat files))
                              files-dir (->> files-with-stats
-                                            (filterv
-                                             (fn [{:keys [type]}]
-                                               (contains? #{"directory" "NSFileTypeDirectory"} type)))
+                                            (filterv #(= (:type %) "directory"))
                                             (mapv :uri))
                              files-result
                              (p/all
                               (->> files-with-stats
-                                   (filter
-                                    (fn [{:keys [type]}]
-                                      (contains? #{"file" "NSFileTypeRegular"} type)))
+                                   (filter #(= (:type %) "file"))
                                    (filter
                                     (fn [{:keys [uri]}]
                                       (some #(string/ends-with? uri %)
                                             [".md" ".markdown" ".org" ".edn" ".css"])))
                                    (mapv
                                     (fn [{:keys [uri] :as file-result}]
-                                      (p/chain
-                                       (read-file-utf8 uri)
-                                       #(js->clj % :keywordize-keys true)
-                                       :data
-                                       #(assoc file-result :content %))))))]
+                                      (p/chain (<read-file-with-utf8 uri)
+                                               #(assoc file-result :content %))))))]
                        (p/recur (concat result files-result)
-                                (concat (rest dirs) files-dir)))))
-          result (js->clj result :keywordize-keys true)]
-    (map (fn [result] (update result :uri clean-uri)) result)))
+                                (concat (rest dirs) files-dir)))))]
+    (js->clj result :keywordize-keys true)))
 
 (defn- contents-matched?
   [disk-content db-content]
@@ -113,7 +128,7 @@
   [repo-dir path ext]
   (let [relative-path (-> (string/replace path repo-dir "")
                           (string/replace (str "." ext) ""))]
-    (str repo-dir backup-dir "/" relative-path)))
+    (util/safe-path-join repo-dir (str backup-dir "/" relative-path))))
 
 (defn- truncate-old-versioned-files!
   "reserve the latest 6 version files"
@@ -122,17 +137,14 @@
           files (js->clj files :keywordize-keys true)
           old-versioned-files (drop 6 (reverse (sort-by :mtime files)))]
     (mapv (fn [file]
-            (.deleteFile Filesystem (clj->js {:path (js/encodeURI (:uri file))})))
+            (.deleteFile Filesystem (clj->js {:path (:uri file)})))
           old-versioned-files)))
 
 (defn backup-file
   [repo-dir path content ext]
   (let [backup-dir (get-backup-dir repo-dir path ext)
         new-path (str backup-dir "/" (string/replace (.toISOString (js/Date.)) ":" "_") "." ext)]
-    (.writeFile Filesystem (clj->js {:data content
-                                     :path new-path
-                                     :encoding (.-UTF8 Encoding)
-                                     :recursive true}))
+    (<write-file-with-utf8 new-path content)
     (truncate-old-versioned-files! backup-dir)))
 
 (defn backup-file-handle-changed!
@@ -149,25 +161,19 @@
         backup-root       (util/safe-path-join repo-dir backup-dir)
         backup-dir-parent (util/node-path.dirname file-path)
         backup-dir-parent (string/replace backup-dir-parent repo-dir "")
-        backup-dir-name   (util/node-path.name file-path)
-        file-extname      (.extname util/node-path file-path)
-        file-root         (util/safe-path-join backup-root backup-dir-parent backup-dir-name)
-        file-path         (util/safe-path-join file-root
-                                               (str (string/replace (.toISOString (js/Date.)) ":" "_") "." (mobile-util/platform) file-extname))]
-    (.writeFile Filesystem (clj->js {:data      content
-                                     :path      (js/encodeURI file-path)
-                                     :encoding  (.-UTF8 Encoding)
-                                     :recursive true}))
-    (truncate-old-versioned-files! (js/encodeURI file-root))))
+        backup-dir-name (util/node-path.name file-path)
+        file-extname (.extname util/node-path file-path)
+        file-root (util/safe-path-join backup-root backup-dir-parent backup-dir-name)
+        file-path (util/safe-path-join file-root
+                                       (str (string/replace (.toISOString (js/Date.)) ":" "_") "." (mobile-util/platform) file-extname))]
+    (<write-file-with-utf8 file-path content)
+    (truncate-old-versioned-files! file-root)))
 
 (defn- write-file-impl!
   [_this repo _dir path content {:keys [ok-handler error-handler old-content skip-compare?]} stat]
   (if skip-compare?
     (p/catch
-     (p/let [result (.writeFile Filesystem (clj->js {:path path
-                                                     :data content
-                                                     :encoding (.-UTF8 Encoding)
-                                                     :recursive true}))]
+     (p/let [result (<write-file-with-utf8 path content)]
        (when ok-handler
          (ok-handler repo path result)))
      (fn [error]
@@ -175,16 +181,12 @@
          (error-handler error)
          (log/error :write-file-failed error))))
 
-    (p/let [disk-content (-> (p/chain (read-file-utf8 path)
-                                      #(js->clj % :keywordize-keys true)
-                                      :data)
-                             (p/catch (fn [error]
-                                        (js/console.error error)
-                                        nil)))
+    ;; Compare with disk content and backup if not equal
+    (p/let [disk-content (<read-file-with-utf8 path)
             disk-content (or disk-content "")
             repo-dir (config/get-local-dir repo)
-            ext (string/lower-case (util/get-file-ext path))
-            db-content (or old-content (db/get-file repo (js/decodeURI path)) "")
+            ext (util/get-file-ext path)
+            db-content (or old-content (db/get-file repo path) "")
             contents-matched? (contents-matched? disk-content db-content)
             pending-writes (state/get-write-chan-length)]
       (cond
@@ -199,10 +201,7 @@
 
         :else
         (->
-         (p/let [result (.writeFile Filesystem (clj->js {:path path
-                                                         :data content
-                                                         :encoding (.-UTF8 Encoding)
-                                                         :recursive true}))
+         (p/let [result (<write-file-with-utf8 path content)
                  mtime (-> (js->clj stat :keywordize-keys true)
                            :mtime)]
            (when-not contents-matched?
@@ -211,7 +210,7 @@
            (p/let [content (if (encrypt/encrypted-db? (state/get-current-repo))
                              (encrypt/decrypt content)
                              content)]
-             (db/set-file-content! repo (js/decodeURI path) content))
+             (db/set-file-content! repo path content))
            (when ok-handler
              (ok-handler repo path result))
            result)
@@ -221,25 +220,19 @@
                       (log/error :write-file-failed error)))))))))
 
 (defn get-file-path [dir path]
-  (let [[dir path] (map #(some-> %
-                                 js/decodeURI)
-                        [dir path])
-        dir (some-> dir (string/replace #"/+$" ""))
-        path (some-> path (string/replace #"^/+" ""))
-        path (cond (nil? path)
-                   dir
+  (let [dir (some-> dir (string/replace #"/+$" ""))
+        path (some-> path (string/replace #"^/+" ""))]
+    (cond (nil? path)
+          dir
 
-                   (nil? dir)
-                   path
+          (nil? dir)
+          path
 
-                   (string/starts-with? path dir)
-                   path
+          (string/starts-with? path dir)
+          path
 
-                   :else
-                   (str dir "/" path))]
-    (if (mobile-util/native-ios?)
-      (js/encodeURI (js/decodeURI path))
-      path)))
+          :else
+          (str dir "/" path))))
 
 (defn- local-container-path?
   "Check whether `path' is logseq's container `localDocumentsPath' on iOS"
@@ -301,11 +294,7 @@
   (read-file [_this dir path _options]
     (let [path (get-file-path dir path)]
       (->
-       (p/let [content (read-file-utf8 path)
-               content (-> (js->clj content :keywordize-keys true)
-                           :data
-                           clj->js)]
-         content)
+       (<read-file-with-utf8 path)
        (p/catch (fn [error]
                   (log/error :read-file-failed error))))))
   (write-file! [this repo dir path content opts]
@@ -332,16 +321,17 @@
                                          }))]
         result)))
   (open-dir [_this _ok-handler]
-    (p/let [_    (when (= (mobile-util/platform) "android") (check-permission-android))
+    (p/let [_ (when (mobile-util/native-android?) (android-check-permission))
             {:keys [path localDocumentsPath]} (-> (.pickFolder mobile-util/folder-picker)
                                                   (p/then #(js->clj % :keywordize-keys true))
                                                   (p/catch (fn [e]
                                                              (js/alert (str e))
-                                                             nil))) ;; NOTE: Can not pick folder, let it crash
+                                                             nil))) ;; NOTE: If pick folder fails, let it crash
             _ (when (and (mobile-util/native-ios?)
                          (not (or (local-container-path? path localDocumentsPath)
                                   (mobile-util/iCloud-container-path? path))))
                 (state/pub-event! [:modal/show-instruction]))
+            _ (js/console.log "Opening or Creating graph at directory: " path)
             files (readdir path)
             files (js->clj files :keywordize-keys true)]
       (into [] (concat [{:path path}] files))))
@@ -350,6 +340,6 @@
   (watch-dir! [_this dir]
     (p/do!
      (.unwatch mobile-util/fs-watcher)
-     (.watch mobile-util/fs-watcher #js {:path dir})))
+     (.watch mobile-util/fs-watcher (clj->js {:path dir}))))
   (unwatch-dir! [_this _dir]
     (.unwatch mobile-util/fs-watcher)))

src/main/frontend/handler/events.cljs

@@ -453,10 +453,11 @@
 (defmethod handle :file-watcher/changed [[_ ^js event]]
   (let [type (.-event event)
         payload (-> event
-                    (js->clj :keywordize-keys true)
-                    (update :path js/decodeURI))]
+                    (js->clj :keywordize-keys true))
+        ;; TODO: remove this
+        payload' (-> payload (update :path js/decodeURI))]
     (fs-watcher/handle-changed! type payload)
-    (sync/file-watch-handler type payload)))
+    (sync/file-watch-handler type payload')))
 
 (defmethod handle :rebuild-slash-commands-list [[_]]
   (page-handler/rebuild-slash-commands-list!))

src/main/frontend/handler/file.cljs

@@ -89,7 +89,7 @@
     (util/electron?)
     (ipc/ipc "backupDbFile" repo-url path db-content content)
 
-    (or (mobile/native-ios?) (mobile/native-android?))
+    (mobile/native-platform?)
     (capacitor-fs/backup-file-handle-changed! repo-url path db-content)
 
     :else
@@ -101,7 +101,9 @@
   (when-let [page-name (:block/name page)]
     (let [current-file (:file/path (db/get-page-file repo-url page-name))]
       (when (not= file current-file)
-       current-file))))
+        (prn ::debug-page-exist-warn repo-url page file)
+        (js/console.trace)
+        current-file))))
 
 (defn- get-delete-blocks [repo-url first-page file]
   (let [delete-blocks (->
@@ -292,7 +294,7 @@
         path (str config/app-name "/" config/metadata-file)
         file-path (str "/" path)
         default-content (if encrypted? "{:db/encrypted? true}" "{}")]
-    (p/let [_ (fs/mkdir-if-not-exists (str repo-dir "/" config/app-name))
+    (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
             file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
       (when-not file-exists?
         (reset-file! repo-url path default-content)))))
@@ -303,7 +305,7 @@
         path (str config/app-name "/" config/pages-metadata-file)
         file-path (str "/" path)
         default-content "{}"]
-    (p/let [_ (fs/mkdir-if-not-exists (str repo-dir "/" config/app-name))
+    (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
             file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
       (when-not file-exists?
         (reset-file! repo-url path default-content)))))

src/main/frontend/handler/repo.cljs

@@ -61,7 +61,7 @@
                               "org" (rc/inline "contents.org")
                               "markdown" (rc/inline "contents.md")
                               "")]
-        (p/let [_ (fs/mkdir-if-not-exists (str repo-dir "/" pages-dir))
+        (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir pages-dir))
                 file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
           (when-not file-exists?
             (file-handler/reset-file! repo-url path default-content)))))))
@@ -73,7 +73,7 @@
         path (str config/app-name "/" config/custom-css-file)
         file-path (str "/" path)
         default-content ""]
-    (p/let [_ (fs/mkdir-if-not-exists (str repo-dir "/" config/app-name))
+    (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
             file-exists? (fs/create-if-not-exists repo-url repo-dir file-path default-content)]
       (when-not file-exists?
         (file-handler/reset-file! repo-url path default-content)))))
@@ -84,7 +84,7 @@
   (let [repo-dir (config/get-repo-dir repo-url)
         path (str (config/get-pages-directory) "/how_to_make_dummy_notes.md")
         file-path (str "/" path)]
-    (p/let [_ (fs/mkdir-if-not-exists (str repo-dir "/" (config/get-pages-directory)))
+    (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (config/get-pages-directory)))
             _file-exists? (fs/create-if-not-exists repo-url repo-dir file-path content)]
       (file-handler/reset-file! repo-url path content))))
 
@@ -110,14 +110,14 @@
 
                     :else
                     default-content)
-          path (str (config/get-journals-directory) "/" file-name "."
-                    (config/get-file-extension format))
+          path (util/safe-path-join (config/get-journals-directory) (str file-name "."
+                                                                         (config/get-file-extension format)))
           file-path (str "/" path)
           page-exists? (db/entity repo-url [:block/name (util/page-name-sanity-lc title)])
           empty-blocks? (db/page-empty? repo-url (util/page-name-sanity-lc title))]
       (when (or empty-blocks? (not page-exists?))
         (p/let [_ (nfs/check-directory-permission! repo-url)
-                _ (fs/mkdir-if-not-exists (str repo-dir "/" (config/get-journals-directory)))
+                _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (config/get-journals-directory)))
                 file-exists? (fs/file-exists? repo-dir file-path)]
           (when-not file-exists?
             (p/let [_ (file-handler/reset-file! repo-url path content)]
@@ -133,9 +133,9 @@
   ([repo-url encrypted?]
    (spec/validate :repos/url repo-url)
    (let [repo-dir (config/get-repo-dir repo-url)]
-     (p/let [_ (fs/mkdir-if-not-exists (str repo-dir "/" config/app-name))
-             _ (fs/mkdir-if-not-exists (str repo-dir "/" config/app-name "/" config/recycle-dir))
-             _ (fs/mkdir-if-not-exists (str repo-dir "/" (config/get-journals-directory)))
+     (p/let [_ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir config/app-name))
+             _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (str config/app-name "/" config/recycle-dir)))
+             _ (fs/mkdir-if-not-exists (util/safe-path-join repo-dir (config/get-journals-directory)))
              _ (file-handler/create-metadata-file repo-url encrypted?)
              _ (create-config-file-if-not-exists repo-url)
              _ (create-contents-file repo-url)

src/main/frontend/mobile/core.cljs

@@ -2,6 +2,7 @@
   (:require ["@capacitor/app" :refer [^js App]]
             ["@capacitor/keyboard" :refer [^js Keyboard]]
             [clojure.string :as string]
+            [promesa.core :as p]
             [frontend.fs.capacitor-fs :as mobile-fs]
             [frontend.handler.editor :as editor-handler]
             [frontend.mobile.deeplink :as deeplink]
@@ -20,8 +21,8 @@
 (defn- ios-init
   "Initialize iOS-specified event listeners"
   []
-  (let [path (mobile-fs/iOS-ensure-documents!)]
-    (println "iOS container path: " path))
+  (p/let [path (mobile-fs/iOS-ensure-documents!)]
+    (println "iOS container path: " (js->clj path)))
 
   (state/pub-event! [:validate-appId])