refactor(fs): rm callback usage from fs protocol

src/main/frontend/fs.cljs

@@ -56,6 +56,9 @@
 
 (defn readdir
   [dir & {:keys [path-only?]}]
+  (when-not path-only?
+    (js/console.error "BUG: (deprecation) path-only? always true")
+    )
   (p/let [result (protocol/readdir (get-fs dir) dir)
           result (bean/->clj result)]
     (let [result (if (and path-only? (map? (first result)))
@@ -162,9 +165,9 @@
     nfs-backend))
 
 (defn open-dir
-  [dir ok-handler]
+  [dir]
   (let [record (get-native-backend)]
-    (p/let [result (protocol/open-dir record dir ok-handler)]
+    (p/let [result (protocol/open-dir record dir)]
       (prn ::open-dir result)
       (if (or (util/electron?)
               (mobile-util/native-platform?))
@@ -178,16 +181,14 @@
           {:path dir :files files})
         result))))
 
-(defn list-files
+(defn get-files
   "List all files in the directory, recursively.
    
    Wrap as {:path string :files []}, using relative path"
-  [dir ok-handler]
+  [dir]
   (let [fs-record (get-native-backend)]
-    (when ok-handler
-      (js/console.warn "ok-handler not nil"))
-    (p/let [result (protocol/list-files fs-record dir ok-handler)]
-      (prn ::list-files (first result) "....")
+    (p/let [result (protocol/get-files fs-record dir)]
+      (prn ::get-files (first result) "....")
       (if (seq result) ;; electron, mobile, nfs
         (let [files result ;; TODO(andelf): rm first item from electron
               dir dir

src/main/frontend/fs/capacitor_fs.cljs

@@ -379,10 +379,9 @@
   (stat [_this fpath]
     (p/chain (.stat Filesystem (clj->js {:path fpath}))
              #(js->clj % :keywordize-keys true)))
-  (open-dir [_this dir _ok-handler]
+  (open-dir [_this dir]
     (open-dir dir))
-  (list-files [_this dir _ok-handler]
-    (prn ::readdir dir)
+  (get-files [_this dir]
     (readdir dir))
   (watch-dir! [_this dir _options]
     (p/do!

src/main/frontend/fs/memory_fs.cljs

@@ -89,9 +89,10 @@
   (stat [_this fpath]
     (let [fpath (fs2-path/url-to-path fpath)]
       (js/window.pfs.stat fpath)))
-  (open-dir [_this _dir _ok-handler]
+  
+  (open-dir [_this _dir]
     nil)
-  (list-files [_this _path-or-handle _ok-handler]
+  (get-files [_this _path-or-handle]
     nil)
   (watch-dir! [_this _dir _options]
     nil)

src/main/frontend/fs/nfs.cljs

@@ -4,7 +4,7 @@
    Rationale:
    nfs-file-handles-cache stores all file & directory handle
    idb stores top-level directory handle
-   readdir/list-files is called by re-index and initial watcher to init all handles"
+   readdir/get-files is called by re-index and initial watcher to init all handles"
   (:require [frontend.fs.protocol :as protocol]
             [frontend.util :as util]
             [clojure.string :as string]
@@ -293,7 +293,7 @@
            :path fpath
            :type (get-attr "type")}))
       (p/rejected "File not exists")))
-  (open-dir [_this _dir _ok-handler]
+  (open-dir [_this _dir]
     (p/let [files (utils/openDirectory #js {:recursive true
                                             :mode "readwrite"}
                                        (fn [path entry]
@@ -336,13 +336,13 @@
       {:path dir-name
        :files files}))
 
-  (list-files [_this dir _ok-handler]
+  (get-files [_this dir]
     (when (string/includes? dir "/")
-      (js/console.error "BUG: list-files(nfs) only accepts repo-dir"))
+      (js/console.error "BUG: get-files(nfs) only accepts repo-dir"))
     (p/let [handle-path (str "handle/" dir)
             handle (get-nfs-file-handle handle-path)
             files (list-and-reload-all-file-handles dir handle)]
-      (prn ::list-files files)
+      (prn ::get-files files)
       files))
 
   (watch-dir! [_this _dir _options]

src/main/frontend/fs/node.cljs

@@ -123,18 +123,18 @@
             stat (p/catch
                   (protocol/stat this fpath)
                   (fn [_e] :not-found))
-            sub-dir (first (util/get-dir-and-basename path)) ;; FIXME: todo dirname
-            _ (protocol/mkdir-recur! this sub-dir)]
+            parent-dir (fs2-path/parent fpath)
+            _ (protocol/mkdir-recur! this parent-dir)]
       (write-file-impl! repo dir path content opts stat)))
   (rename! [_this _repo old-path new-path]
     (ipc/ipc "rename" old-path new-path))
   (stat [_this fpath]
     (-> (ipc/ipc "stat" fpath)
         (p/then bean/->clj)))
-  (open-dir [_this dir _ok-handler]
+  (open-dir [_this dir]
     (p/then (open-dir dir)
             bean/->clj))
-  (list-files [_this dir _ok-handler]
+  (get-files [_this dir]
     (-> (ipc/ipc "getFiles" dir)
         (p/then bean/->clj)))
   (watch-dir! [_this dir options]

src/main/frontend/fs/protocol.cljs

@@ -5,8 +5,12 @@
 (defprotocol Fs
   (mkdir! [this dir])
   (mkdir-recur! [this dir])
-  ;; TODO(andelf): clarify the return value. How is this different from `list-files`?
-  (readdir [this dir])
+  ;; TODO(andelf): clarify the return value. How is this different from `get-files`?
+  (readdir [this dir]
+    "Read directory and return list of files. Won't read file out.
+     Used by initial watcher, version files of Logseq Sync.
+     
+     => [string]")
   (unlink! [this repo path opts])
   ;; FIXME(andelf): remove this API? since the only usage is plugin API
   (rmdir! [this dir])
@@ -16,10 +20,14 @@
   (copy! [this repo old-path new-path])
   (stat [this path]
     "=> {:type string :size number :mtime number}")
-  (open-dir [this dir ok-handler]
-    "=> {:path string :files [{...}]}")
-  (list-files [this dir ok-handler]
-    "=> [{:path string :content string}] (absolute path)")
+  (open-dir [this dir]
+    "Open a directory and return the files in it.
+     
+     => {:path string :files [{...}]}")
+  (get-files [this dir]
+    "Almost the same as `open-dir`. For returning files.
+     
+     => [{:path string :content string}] (absolute path)")
   (watch-dir! [this dir options])
   (unwatch-dir! [this dir])
   ;; Ensure the dir is watched, window agnostic.

src/main/frontend/fs/watcher_handler.cljs

@@ -128,6 +128,7 @@
                         (map first)
                         (filter #(string/starts-with? % (config/get-repo-dir graph))))]
       (p/let [files (fs/readdir dir :path-only? true)
+              _ (prn ::read-files files)
               files (map #(fs2-path/relative-path dir %) files)       ;; FIXME(andelf): readdir returns full paths
               files (remove #(fs-util/ignored-path? dir %) files)]
         (let [deleted-files (set/difference (set db-files) (set files))]

src/main/frontend/fs2/path.cljs

@@ -104,8 +104,6 @@
   '..' and '.' normalization."
   [& segments]
   (let [segments (remove nil? segments) ;; handle (path-join nil path)
-        ; _ (prn ::uri-join-seg segments)
-        ; _ (js/console.trace)
         segments (map #(string/replace % #"[/\\]+" "/") segments)
         ;; a fix for clojure.string/split
         split-fn (fn [s]
@@ -155,9 +153,8 @@
 (defn path-join
   "Join path segments, or URL base and path segments"
   [base & segments]
-  (prn ::join base segments)
   (when (string/blank? base)
-    (prn ::SHOULD-NOT-JOIN-EMPTY)
+    (js/console.error "BUG: SHOULD-NOT-JOIN-EMPTY")
     (js/console.trace))
 
   (if (is-file-url base)
@@ -222,12 +219,9 @@
   "Get relative path from base path.
    Works for both path and URL."
   [base-path sub-path]
-  (prn :rel-path base-path sub-path)
-  ;; (js/console.trace)
   (let [base-path (path-normalize base-path)
         sub-path (path-normalize sub-path)
         is-url? (is-file-url base-path)]
-    (prn :rel-path base-path sub-path)
     (if (string/starts-with? sub-path base-path)
       (if is-url?
         (gp-util/safe-decode-uri-component (string/replace (subs sub-path (count base-path)) #"^/+", ""))
@@ -240,7 +234,7 @@
             base-segs (drop (count common-segs) base-segs)
             remain-segs (drop (count common-segs) path-segs)
             base-prefix (apply str (repeat (max 0 (dec (count base-segs))) "../"))]
-        (js/console.error (js/Error. "buggy"))
+        (js/console.error (js/Error. "buggy relative-path"))
         #_{:clj-kondo/ignore [:path-invalid-construct/string-join]}
         (if is-url?
           (gp-util/safe-decode-uri-component (str base-prefix (string/join "/" remain-segs)))

src/main/frontend/handler/repo.cljs

@@ -46,8 +46,6 @@
           contents-file-exist? (some #(fs/file-exists? repo-dir %) [org-path md-path])]
     (when-not contents-file-exist?
       (let [format (state/get-preferred-format)
-            ;;path (str pages-dir "/contents."
-            ;;          (config/get-file-extension format))
             file-rpath (str "pages/" "contents." (config/get-file-extension format))
             default-content (case (name format)
                               "org" (rc/inline "contents.org")
@@ -289,10 +287,9 @@
   [repo-url {:keys [diffs file-objs refresh? new-graph? empty-graph?]}]
   (spec/validate :repos/url repo-url)
   (route-handler/redirect-to-home!)
-  (prn ::load----file-objs file-objs)
   (state/set-parsing-state! {:graph-loading? true})
   (let [repo-dir (config/get-local-dir repo-url)
-        _ (prn ::repo-dir repo-dir)
+        _ (prn ::load-to-db-repo-dir repo-dir)
         config (or (when-let [content (some-> (first (filter #(= (config/get-repo-config-path repo-url) (:file/path %)) file-objs))
                                               :file/content)]
                      (repo-config-handler/read-repo-config content))

src/main/frontend/handler/web/nfs.cljs

@@ -44,7 +44,6 @@
         (p/resolved files))
       (p/resolved files))))
 
-;; TODO: this should be handled in fs layer
 (defn- ->db-files
   ;; TODO(andelf): rm nfs? parameter
   [result nfs?]
@@ -122,8 +121,7 @@
   "Read files from directory and setup repo (for the first time setup a repo)"
   ([ok-handler] (ls-dir-files-with-handler! ok-handler nil))
   ([ok-handler {:keys [on-open-dir dir-result-fn picked-root-fn dir]}]
-   (let [path-handles (atom {})
-         electron? (util/electron?)
+   (let [electron? (util/electron?)
          mobile-native? (mobile-util/native-platform?)
          nfs? (and (not electron?)
                    (not mobile-native?))
@@ -133,12 +131,8 @@
      (->
       (p/let [_ (prn :xxx-dir-result-fn dir-result-fn)
               result (if (fn? dir-result-fn)
-                       (dir-result-fn {:path-handles path-handles :nfs? nfs?})
-                       ;; TODO: rm callback
-                       (fs/open-dir dir
-                                    (fn [path handle]
-                                      (comment when nfs?
-                                               (swap! path-handles assoc path handle)))))
+                       (dir-result-fn {:nfs? nfs?})
+                       (fs/open-dir dir))
               _ (when (fn? on-open-dir)
                   (prn ::calling-on-open-dir-fn)
                   (on-open-dir result))
@@ -146,7 +140,7 @@
               _ (when (fn? picked-root-fn) (picked-root-fn root-handle))
               dir-name root-handle
               repo (str config/local-db-prefix root-handle)]
-        
+
         (state/set-loading-files! repo true)
         (when-not (or (state/home?) (state/setups-picker?))
           (route-handler/redirect-to-home! false))
@@ -167,25 +161,6 @@
                             ;; NOTE: filter, in case backend does not handle this
                             (remove-ignore-files dir-name nfs?))
                   _ (prn ::remain-files files)
-                  _ (comment when nfs?
-                      ;; only for browserfs
-                             (let [file-paths (set (map :file/path files))]
-                               (swap! path-handles (fn [handles]
-                                                     (->> handles
-                                                          (filter (fn [[path _handle]]
-                                                                    (or
-                                                                     (contains? file-paths
-                                                                                (string/replace-first path (str dir-name "/") ""))
-                                                                     (let [last-part (last (string/split path "/"))]
-                                                                       (contains? #{config/app-name
-                                                                                    gp-config/default-draw-directory
-                                                                                    (config/get-journals-directory)
-                                                                                    (config/get-whiteboards-directory)
-                                                                                    (config/get-pages-directory)}
-                                                                                  last-part)))))
-                                                          (into {})))))
-
-                             (set-files! @path-handles))
                   markup-files (filter-markup-and-built-in-files files)]
             (-> files
                 (p/then (fn [result]
@@ -233,12 +208,8 @@
   ([path] (ls-dir-files-with-path! path nil))
   ([path opts]
    (when-let [dir-result-fn
-              (and path (fn [{:keys [path-handles nfs?]}]
-                          (p/let [files-result (fs/list-files
-                                                path
-                                                (fn [path handle]
-                                                  (when nfs?
-                                                    (swap! path-handles assoc path handle))))]
+              (and path (fn [{:keys [nfs?]}]
+                          (p/let [files-result (fs/get-files path)]
                             [path (:files files-result)])))]
      (ls-dir-files-with-handler!
       (:ok-handler opts)
@@ -265,13 +236,14 @@
                                 (not= (:file/content (get-file-f old-files path))
                                       (:file/content (get-file-f new-files path)))))
                       (set))]
+    (prn ::compute-diffs :added (count added) :modified (count modified) :deleted (count deleted))
     {:added    added
      :modified modified
      :deleted  deleted}))
 
 (defn- handle-diffs!
   "Compute directory diffs and handle them."
-  [repo nfs? old-files new-files handle-path path-handles re-index? ok-handler]
+  [repo nfs? old-files new-files re-index? ok-handler]
   (prn ::handle-diff repo old-files new-files)
   (let [get-last-modified-at (fn [path] (some (fn [file]
                                                 (when (= path (:file/path file))
@@ -281,21 +253,7 @@
         {:keys [added modified deleted]} (compute-diffs old-files new-files)
         ;; Use the same labels as isomorphic-git
         rename-f (fn [typ col] (mapv (fn [file] {:type typ :path file :last-modified-at (get-last-modified-at file)}) col))
-        _ (when (and nfs? (seq deleted))
-            (let [deleted (doall
-                           (-> (map (fn [path] (if (= "/" (first path))
-                                                 path
-                                                 (str "/" path))) deleted)
-                               (distinct)))]
-              (p/all (map (fn [path]
-                            (let [handle-path (str handle-path path)]
-                              (idb/remove-item! handle-path)
-                              (nfs/remove-nfs-file-handle! handle-path))) deleted))))
-        added-or-modified (set (concat added modified))
-        _ (when (and nfs? (seq added-or-modified))
-            (p/all (map (fn [path]
-                          (when-let [handle (get @path-handles path)]
-                            (idb/set-item! (str handle-path path) handle))) added-or-modified)))]
+        added-or-modified (set (concat added modified))]
     (-> (p/all (map (fn [path]
                       (when-let [file (get-file-f path new-files)]
                         (p/let [content (if nfs?
@@ -338,7 +296,6 @@
     (let [old-files (db/get-files-full repo)
           repo-dir (config/get-local-dir repo)
           handle-path (str "handle/" repo-dir)
-          path-handles (atom {})
           electron? (util/electron?)
           mobile-native? (mobile-util/native-platform?)
           nfs? (and (not electron?)
@@ -348,41 +305,33 @@
       (->
        (p/let [handle (when-not electron? (idb/get-item handle-path))]
          (prn ::handle handle)
-         (when (or handle electron? mobile-native?)   ; electron doesn't store the file handle
-           (p/let [_ (when handle (nfs/verify-permission repo true))
-                   local-files-result
-                   (fs/list-files repo-dir
-                                  (fn [path handle]
-                                    (comment when false ; nfs?
-                                      (swap! path-handles assoc path handle)))
-                                  )
+         (when (or handle electron? mobile-native?)
+           (p/let [_ (when nfs? (nfs/verify-permission repo true))
+                   local-files-result (fs/get-files repo-dir)
                    _ (prn ::reading-local-fils local-files-result)
-                   new-local-files (-> (->db-files (:files local-files-result) nfs?)
+                   new-files (-> (->db-files (:files local-files-result) nfs?)
                                        (remove-ignore-files repo-dir nfs?))
-                   _ (prn ::new-local-files)
+                   _ (prn ::new-local-files new-files)
                   ;; new-global-files (if (and (config/global-config-enabled?)
                    ;;                          ;; Hack until we better understand failure in frontend.handler.file/alter-file
                     ;;                         (global-config-handler/global-config-dir-exists?))
-                     ;;                 (p/let [global-files-result (fs/list-files
+                     ;;                 (p/let [global-files-result (fs/get-files
                       ;;                                             (global-config-handler/global-config-dir)
                        ;;                                            (constantly nil))
                         ;;                      global-files (-> (->db-files (global-config-handler/global-config-dir) global-files-result)
                          ;;                                      (remove-ignore-files (global-config-handler/global-config-dir) nfs?))]
                           ;;              global-files)
-                           ;;           (p/resolved []))
-                   new-files new-local-files ;; (concat new-local-files new-global-files)
-
+                           ;;           (p/resolved [])) 
                    _ (comment when nfs?
-                       (let [file-paths (set (map :file/path new-files))]
-                         (swap! path-handles (fn [handles]
-                                               (->> handles
-                                                    (filter (fn [[path _handle]]
-                                                              (contains? file-paths
-                                                                         (string/replace-first path (str dir-name "/") ""))))
-                                                    (into {})))))
-                       (set-files! @path-handles))]
-             (prn ::going-to-handle new-files)
-             (handle-diffs! repo nfs? old-files new-files handle-path path-handles re-index? ok-handler))))
+                              (let [file-paths (set (map :file/path new-files))]
+                                (swap! path-handles (fn [handles]
+                                                      (->> handles
+                                                           (filter (fn [[path _handle]]
+                                                                     (contains? file-paths
+                                                                                (string/replace-first path (str dir-name "/") ""))))
+                                                           (into {})))))
+                              (set-files! @path-handles))]
+             (handle-diffs! repo nfs? old-files new-files re-index? ok-handler))))
        (p/catch (fn [error]
                   (log/error :nfs/load-files-error repo)
                   (log/error :exception error)))