refactor(fs): rm redundent return date from fs protocol

fix(fs): asset url loading under nfs backend

src/main/frontend/components/block.cljs

@@ -386,7 +386,9 @@
       (p/then (editor-handler/make-asset-url href) #(reset! src %)))
 
     (when @src
-      (let [ext (keyword (util/get-file-ext @src))
+      ;; NOTE(andelf): Under nfs context, src might be a bare blob:http://..../uuid URI without ext info
+      (let [ext (keyword (or (util/get-file-ext @src)
+                             (util/get-file-ext href)))
             repo (state/get-current-repo)
             repo-dir (config/get-repo-dir repo)
             path (str repo-dir href)
@@ -395,8 +397,7 @@
                        (when (mobile-util/native-platform?)
                          ;; File URL must be legal, so filename muse be URI-encoded
                          (let [[rel-dir basename] (util/get-dir-and-basename href)
-                               basename (js/encodeURIComponent basename)
-                               asset-url (str repo-dir rel-dir "/" basename)]
+                               asset-url (fs2-path/path-join repo-dir rel-dir basename)]
                            (.share Share (clj->js {:url asset-url
                                                    :title "Open file with your favorite app"})))))]
 

src/main/frontend/fs.cljs

@@ -24,10 +24,6 @@
 (defonce node-backend (node/->Node))
 (defonce mobile-backend (capacitor-fs/->Capacitorfs))
 
-(defn local-db?
-  [dir]
-  (and (string? dir)
-       (config/local-db? (subs dir 1))))
 
 (defn get-fs
   [dir]
@@ -84,8 +80,7 @@
   [repo dir rpath content opts]
   (when content
     (let [path (gp-util/path-normalize rpath)
-          fs-record (get-fs dir)
-          _ (prn ::debug-fs fs-record)]
+          fs-record (get-fs dir)]
       (->
        (p/let [opts (assoc opts
                            :error-handler

src/main/frontend/fs/capacitor_fs.cljs

@@ -93,8 +93,33 @@
                  (js/console.error "stat Error: " path ": " error)
                  nil))))
 
-(defn readdir
-  "readdir recursively"
+(defn- get-file-paths
+  "get all file paths recursively"
+  [path]
+  (p/let [result (p/loop [result []
+                          dirs [path]]
+                   (if (empty? dirs)
+                     result
+                     (p/let [d (first dirs)
+                             files (<readdir d)
+                             files (->> files
+                                        (remove (fn [{:keys [name  type]}]
+                                                  (or (string/starts-with? name ".")
+                                                      (and (= type "directory")
+                                                           (or (= name "bak")
+                                                               (= name "version-files")))))))
+                             files-dir (->> files
+                                            (filterv #(= (:type %) "directory"))
+                                            (mapv :uri))
+                             paths-result (->> files
+                                               (filterv #(= (:type %) "file"))
+                                               (mapv :uri))]
+                       (p/recur (concat result paths-result)
+                                (concat (rest dirs) files-dir)))))]
+    result))
+
+(defn- get-files
+  "get all files recursively"
   [path]
   (p/let [result (p/loop [result []
                           dirs [path]]
@@ -150,7 +175,7 @@
   "reserve the latest 6 version files"
   [dir]
   (->
-   (p/let [files (readdir dir)
+   (p/let [files (get-files dir)
            files (js->clj files :keywordize-keys true)
            old-versioned-files (drop 6 (reverse (sort-by :mtime files)))]
      (mapv (fn [file]
@@ -301,7 +326,7 @@
                   (ios-force-include-private path)
                   path)
           _ (js/console.log "Opening or Creating graph at directory: " path)
-          files (readdir path)]
+          files (get-files path)]
     ;; FIXME: wrong stucture returned
     (into [] (concat [{:path path}] files))))
 
@@ -332,7 +357,7 @@
                                              :error error})))))
   (readdir [_this dir]                  ; recursive
     (let [dir (fs2-path/path-normalize dir)]
-      (readdir dir)))
+      (get-file-paths dir)))
   (unlink! [this repo fpath _opts]
     (p/let [_ (prn ::unlink fpath)
             repo-dir (config/get-local-dir repo)
@@ -382,7 +407,7 @@
   (open-dir [_this dir]
     (open-dir dir))
   (get-files [_this dir]
-    (readdir dir))
+    (get-files dir))
   (watch-dir! [_this dir _options]
     (p/do!
      (.unwatch mobile-util/fs-watcher)

src/main/frontend/fs/memory_fs.cljs

@@ -1,7 +1,7 @@
 (ns ^:no-doc frontend.fs.memory-fs
   "Memory FS backed by lightning-fs
    
-   denoted by `memory://`"
+   Paths are denoted by `memory://`. No open-dir/get-files support."
   (:require [cljs-bean.core :as bean]
             [frontend.db :as db]
             [frontend.fs.protocol :as protocol]
@@ -57,7 +57,6 @@
     (when js/window.pfs
       (let [fpath (fs2-path/url-to-path dir)]
         (-> (<readdir fpath)
-            ;; (p/then bean/->clj)
             (p/then (fn [rpaths]
                       (prn ::debug rpaths)
                       (mapv #(fs2-path/path-join "memory://" %) rpaths)))))))

src/main/frontend/fs/nfs.cljs

@@ -88,7 +88,7 @@
 (defn- contents-matched?
   [disk-content db-content]
   (when (and (string? disk-content) (string? db-content))
-    (p/resolved (= (string/trim disk-content) (string/trim db-content)))))
+    (= (string/trim disk-content) (string/trim db-content))))
 
 (defn- await-permission-granted
   "Guard against File System Access API permission, avoiding early access before granted"
@@ -107,8 +107,41 @@
                                     (reject false))
                                   100000)))))
 
-(defn- list-and-reload-all-file-handles [root-dir root-handle]
-  (prn ::list-and-reload-all-file-handles root-handle)
+(defn await-get-nfs-file-handle
+  "for accessing File handle outside, ensuring user granted."
+  [repo handle-path]
+  (p/do!
+   (await-permission-granted repo)
+   (get-nfs-file-handle handle-path)))
+
+(defn- readdir-and-reload-all-handles 
+  "Return list of filenames"
+  [root-dir root-handle] 
+  (p/let [files (utils/getFiles root-handle
+                                true
+                                (fn [path entry]
+                                  (let [handle-path (str "handle/" path)]
+                                    ;; Same for all handles here, even for directories and ignored directories(for backing up)
+                                    ;; FileSystemDirectoryHandle or FileSystemFileHandle
+                                    (add-nfs-file-handle! handle-path entry))))]
+    (->> files
+         (remove  (fn [file]
+                    (let [rpath (string/replace-first (.-webkitRelativePath file) (str root-dir "/") "")
+                          ext (util/get-file-ext rpath)]
+                      (or  (string/blank? rpath)
+                           (string/starts-with? rpath ".")
+                           (string/starts-with? rpath "logseq/bak")
+                           ; (string/starts-with? rpath "logseq/version-files")
+                           (not (contains? #{"md" "org" "excalidraw" "edn" "css"} ext))))))
+         (map (fn [file]
+                (-> (.-webkitRelativePath file)
+                    gp-util/path-normalize))))))
+
+
+(defn- get-files-and-reload-all-handles
+  "Return list of file objects"
+  [root-dir root-handle]
+  (prn ::get-files-and-reload-all-file-handles root-handle)
   (p/let [files (utils/getFiles root-handle
                                 true
                                 (fn [path entry]
@@ -173,10 +206,10 @@
                        (idb/get-item handle-path))
             _ (when handle
                 (verify-handle-permission handle true))
-            files (if (string/includes? dir "/")
+            fpaths (if (string/includes? dir "/")
                     (js/console.error "ERROR: unimpl")
-                    (list-and-reload-all-file-handles dir handle))]
-      files))
+                    (readdir-and-reload-all-handles dir handle))]
+      fpaths))
 
   (unlink! [this repo path _opts]
     (let [[dir basename] (util/get-dir-and-basename path)
@@ -229,7 +262,6 @@
                   disk-content (.text local-file)
                   db-content (db/get-file repo path)
                   contents-matched? (contents-matched? disk-content db-content)]
-            (prn ::file-exist file-handle disk-content db-content)
             (if (and
                  (not (string/blank? db-content))
                  (not (:skip-compare? opts))
@@ -341,7 +373,7 @@
       (js/console.error "BUG: get-files(nfs) only accepts repo-dir"))
     (p/let [handle-path (str "handle/" dir)
             handle (get-nfs-file-handle handle-path)
-            files (list-and-reload-all-file-handles dir handle)]
+            files (get-files-and-reload-all-handles dir handle)]
       (prn ::get-files files)
       files))
 

src/main/frontend/fs/node.cljs

@@ -61,8 +61,7 @@
            (not contents-matched?)
            (not (contains? #{"excalidraw" "edn" "css"} ext))
            (not (string/includes? rpath "/.recycle/")))
-          (do
-            (state/pub-event! [:file/not-matched-from-disk rpath disk-content content]))
+          (state/pub-event! [:file/not-matched-from-disk rpath disk-content content])
 
           :else
           (->
@@ -90,15 +89,6 @@
     (prn ::open-dir result)
     result))
 
-(defn- <ensure-dir!
-  [fs dir]
-  (protocol/mkdir-recur! fs dir))
-
-(defn- <exists
-  [fs path]
-
-  )
-
 (defrecord Node []
   protocol/Fs
   (mkdir! [_this dir]

src/main/frontend/fs/protocol.cljs

@@ -20,17 +20,17 @@
   (copy! [this repo old-path new-path])
   (stat [this path]
     "=> {:type string :size number :mtime number}")
+
+  ;; The following APIs are optional
   (open-dir [this dir]
     "Open a directory and return the files in it.
+     Used by open a new graph.
      
      => {:path string :files [{...}]}")
   (get-files [this dir]
     "Almost the same as `open-dir`. For returning files.
+     Used by re-index/refresh.
      
      => [{:path string :content string}] (absolute path)")
   (watch-dir! [this dir options])
-  (unwatch-dir! [this dir])
-  ;; Ensure the dir is watched, window agnostic.
-  ;; Implementation should handle the actual watcher's construction / destruction.
-  ;; So shouldn't consider `unwatch-dir!`.
-  )
+  (unwatch-dir! [this dir]))

src/main/frontend/handler/editor.cljs

@@ -14,6 +14,7 @@
             [frontend.format.block :as block]
             [frontend.format.mldoc :as mldoc]
             [frontend.fs :as fs]
+            [frontend.fs.nfs :as nfs]
             [frontend.handler.block :as block-handler]
             [frontend.handler.common :as common-handler]
             [frontend.handler.export.text :as export-text]
@@ -22,7 +23,6 @@
             [frontend.handler.repeated :as repeated]
             [frontend.handler.route :as route-handler]
             [frontend.handler.assets :as assets-handler]
-            [frontend.idb :as idb]
             [frontend.mobile.util :as mobile-util]
             [frontend.modules.outliner.core :as outliner-core]
             [frontend.modules.outliner.transaction :as outliner-tx]
@@ -55,8 +55,7 @@
             [logseq.graph-parser.util.page-ref :as page-ref]
             [promesa.core :as p]
             [rum.core :as rum]
-            [frontend.fs2.path :as fs2-path]
-            [frontend.db.model :as model]))
+            [frontend.fs2.path :as fs2-path]))
 
 ;; FIXME: should support multiple images concurrently uploading
 
@@ -1372,6 +1371,7 @@
     (commands/restore-state)))
 
 (defn get-asset-file-link
+  "Link text for inserting to markdown/org"
   [format url file-name image?]
   (let [pdf?   (and url (string/ends-with? (string/lower-case url) ".pdf"))
         media? (and url (or (config/ext-of-audio? url)
@@ -1400,7 +1400,7 @@
 (defn save-assets!
   "Save incoming(pasted) assets to assets directory.
 
-   Returns: [filename file-obj file-fpath matched-alias]"
+   Returns: [file-rpath file-obj file-fpath matched-alias]"
   ([_ repo files]
    (p/let [[repo-dir assets-dir] (ensure-assets-dir! repo)]
      (save-assets! repo repo-dir assets-dir files
@@ -1412,7 +1412,7 @@
                                          (string/replace "/" "_"))
                            file-name (str file-base "_" (.now js/Date) "_" index)]
                        (string/replace file-name #"_+" "_"))))))
-  ([repo dir path files gen-filename]
+  ([repo repo-dir asset-dir-rpath files gen-filename]
    (p/all
     (for [[index ^js file] (map-indexed vector files)]
       ;; WARN file name maybe fully qualified path when paste file
@@ -1425,36 +1425,36 @@
                                                                     (count ext-full))) ext-full ext-base])
                                             ["" "" ""])
             filename  (str (gen-filename index file-stem) ext-full)
-            filename  (str path "/" filename)
+            file-rpath  (str asset-dir-rpath "/" filename)
             matched-alias (assets-handler/get-matched-alias-by-ext ext-base)
-            filename (cond-> filename
-                       (not (nil? matched-alias))
-                       (string/replace #"^[.\/\\]*assets[\/\\]+" ""))
-            dir (or (:dir matched-alias) dir)]
-
+            file-rpath (cond-> file-rpath
+                         (not (nil? matched-alias))
+                         (string/replace #"^[.\/\\]*assets[\/\\]+" ""))
+            dir (or (:dir matched-alias) repo-dir)]
         (if (util/electron?)
           (let [from (not-empty (.-path file))]
 
-            (js/console.debug "Debug: Copy Asset #" dir filename from)
-
-            (-> (js/window.apis.copyFileToAssets dir filename from)
+            (js/console.debug "Debug: Copy Asset #" dir file-rpath from)
+            (-> (js/window.apis.copyFileToAssets dir file-rpath from)
                 (p/then
                  (fn [dest]
-                   [filename
+                   [file-rpath
                     (if (string? dest) (js/File. #js[] dest) file)
-                    (.join util/node-path dir filename)
+                    (.join util/node-path dir file-rpath)
                     matched-alias]))
                 (p/catch #(js/console.error "Debug: Copy Asset Error#" %))))
 
-          (p/then (fs/write-file! repo dir filename (.stream file) nil)
-                  #(p/resolved [filename file nil matched-alias]))))))))
+          (p/do! (js/console.debug "Debug: Writing Asset #" dir file-rpath)
+                 (fs/write-file! repo dir file-rpath (.stream file) nil)
+                 [file-rpath file (fs2-path/path-join dir file-rpath) matched-alias])))))))
 
 (defonce *assets-url-cache (atom {}))
 
 (defn make-asset-url
   "Make asset URL for UI element, to fill img.src"
   [path] ;; path start with "/assets" or compatible for "../assets"
-  (if config/publishing? path
+  (if config/publishing?
+    path
     (let [repo      (state/get-current-repo)
           repo-dir  (config/get-repo-dir repo)
           path      (string/replace path "../" "/")
@@ -1475,12 +1475,14 @@
         (mobile-util/native-platform?)
         (mobile-util/convert-file-src full-path)
 
-        :else
-        (let [handle-path (str "handle" full-path)
+        :else ;; nfs
+        (let [handle-path (str "handle/" full-path)
               cached-url  (get @*assets-url-cache (keyword handle-path))]
           (if cached-url
             (p/resolved cached-url)
-            (p/let [handle (idb/get-item handle-path)
+            ;; Loading File from handle cache
+            ;; Use await file handle, to ensure all handles are loaded.
+            (p/let [handle (nfs/await-get-nfs-file-handle repo handle-path)
                     file   (and handle (.getFile handle))]
               (when file
                 (p/let [url (js/URL.createObjectURL file)]