feat: native filesystem api integration WIP

src/main/frontend/components/header.cljs

@@ -14,7 +14,8 @@
             [frontend.components.svg :as svg]
             [frontend.components.repo :as repo]
             [frontend.components.page :as page]
-            [frontend.components.search :as search]))
+            [frontend.components.search :as search]
+            [frontend.handler.web.nfs :as nfs]))
 
 (rum/defc logo < rum/reactive
   [{:keys [white?]}]
@@ -123,6 +124,11 @@
 
      (new-block-mode)
 
+     [:a.text-sm.font-medium.login.opacity-70.hover:opacity-100.mr-4
+      {:on-click (fn []
+                   (nfs/ls-dir-files))}
+      "Open a database"]
+
      (when (and (not logged?)
                 (not config/publishing?))
        [:a.text-sm.font-medium.login.opacity-70.hover:opacity-100

src/main/frontend/config.cljs

@@ -271,3 +271,13 @@
 (def markers
   #{"now" "later" "todo" "doing" "done" "wait" "waiting"
     "canceled" "cancelled" "started" "in-progress"})
+
+(defonce local-db-prefix "logseq-local-")
+
+(defn local-db?
+  [s]
+  (string/starts-with? s local-db-prefix))
+
+(defn get-local-dir
+  [s]
+  (string/replace s local-db-prefix ""))

src/main/frontend/db.cljs

@@ -11,7 +11,7 @@
             [clojure.set :as set]
             [frontend.utf8 :as utf8]
             [frontend.config :as config]
-            ["localforage" :as localforage]
+            [goog.object :as gobj]
             [promesa.core :as p]
             [cljs.reader :as reader]
             [cljs-time.core :as t]
@@ -21,35 +21,13 @@
             [frontend.extensions.sci :as sci]
             [frontend.db-schema :as db-schema]
             [clojure.core.async :as async]
-            [frontend.storage :as storage]
-            [lambdaisland.glogi :as log]
-            [goog.object :as gobj]))
-
-;; offline db
-(def store-name "dbs")
-(.config localforage
-         #js
-          {:name "logseq-datascript"
-           :version 1.0
-           :storeName store-name})
-
-(defonce localforage-instance (.createInstance localforage store-name))
+            [lambdaisland.glogi :as log]))
 
 ;; Query atom of map of Key ([repo q inputs]) -> atom
 ;; TODO: replace with LRUCache, only keep the latest 20 or 50 items?
 (defonce query-state (atom {}))
 
-(defn clear-idb!
-  []
-  (p/let [_ (.clear localforage-instance)
-          dbs (js/window.indexedDB.databases)]
-    (doseq [db dbs]
-      (js/window.indexedDB.deleteDatabase (gobj/get db "name")))))
-
-(defn clear-local-storage-and-idb!
-  []
-  (storage/clear)
-  (clear-idb!))
+(defonce async-chan (atom nil))
 
 (defn get-repo-path
   [url]
@@ -70,11 +48,11 @@
 
 (defn remove-db!
   [repo]
-  (.removeItem localforage-instance (datascript-db repo)))
+  (idb/remove-item! (datascript-db repo)))
 
 (defn remove-files-db!
   [repo]
-  (.removeItem localforage-instance (datascript-files-db repo)))
+  (idb/remove-item! (datascript-files-db repo)))
 
 (def react util/react)
 
@@ -126,11 +104,10 @@
 
 ;; persisting DB between page reloads
 (defn persist [repo db files-db?]
-  (.setItem localforage-instance
-            (if files-db?
+  (let [key (if files-db?
               (datascript-files-db repo)
-              (datascript-db repo))
-            (db->string db)))
+              (datascript-db repo))]
+    (idb/set-item! key (db->string db))))
 
 (defn reset-conn! [conn db]
   (reset! conn db))
@@ -968,6 +945,8 @@
                     contents))
         all-data (-> (concat delete-files delete-blocks files blocks-pages)
                      (util/remove-nils))]
+    (prn {:repo-url repo-url
+          :all-data all-data})
     (transact! repo-url all-data)))
 
 (defn get-block-by-uuid
@@ -1875,16 +1854,20 @@
       config)))
 
 (defn start-db-conn!
-  [me repo]
-  (let [files-db-name (datascript-files-db repo)
-        files-db-conn (d/create-conn db-schema/files-db-schema)
-        db-name (datascript-db repo)
-        db-conn (d/create-conn db-schema/schema)]
-    (swap! conns assoc files-db-name files-db-conn)
-    (swap! conns assoc db-name db-conn)
-    (d/transact! db-conn [{:schema/version db-schema/version}])
-    (when me
-      (d/transact! db-conn [(me-tx (d/db db-conn) me)]))))
+  ([me repo]
+   (start-db-conn! me repo {}))
+  ([me repo {:keys [db-type]}]
+   (let [files-db-name (datascript-files-db repo)
+         files-db-conn (d/create-conn db-schema/files-db-schema)
+         db-name (datascript-db repo)
+         db-conn (d/create-conn db-schema/schema)]
+     (swap! conns assoc files-db-name files-db-conn)
+     (swap! conns assoc db-name db-conn)
+     (d/transact! db-conn [(cond-> {:schema/version db-schema/version}
+                             db-type
+                             (assoc :db/type db-type))])
+     (when me
+       (d/transact! db-conn [(me-tx (d/db db-conn) me)])))))
 
 (defn restore!
   [{:keys [repos] :as me} restore-config-handler]
@@ -1895,27 +1878,30 @@
              db-name (datascript-files-db repo)
              db-conn (d/create-conn db-schema/files-db-schema)]
          (swap! conns assoc db-name db-conn)
-         (p/let [stored (-> (.getItem localforage-instance db-name)
-                            (p/then (fn [result]
-                                      result))
-                            (p/catch (fn [error]
-                                       nil)))
-                 _ (when stored
-                     (let [stored-db (string->db stored)
-                           attached-db (d/db-with stored-db [(me-tx stored-db me)])]
-                       (reset-conn! db-conn attached-db)))
-                 db-name (datascript-db repo)
-                 db-conn (d/create-conn db-schema/schema)
-                 _ (d/transact! db-conn [{:schema/version db-schema/version}])
-                 _ (swap! conns assoc db-name db-conn)
-                 stored (.getItem localforage-instance db-name)
-                 _ (if stored
-                     (let [stored-db (string->db stored)
-                           attached-db (d/db-with stored-db [(me-tx stored-db me)])]
-                       (reset-conn! db-conn attached-db))
-                     (when logged?
-                       (d/transact! db-conn [(me-tx (d/db db-conn) me)])))]
-           (restore-config-handler repo)))))))
+         (->
+          (p/let [stored (-> (idb/get-item db-name)
+                             (p/then (fn [result]
+                                       result))
+                             (p/catch (fn [error]
+                                        nil)))
+                  _ (when stored
+                      (let [stored-db (string->db stored)
+                            attached-db (d/db-with stored-db [(me-tx stored-db me)])]
+                        (reset-conn! db-conn attached-db)))
+                  db-name (datascript-db repo)
+                  db-conn (d/create-conn db-schema/schema)
+                  _ (d/transact! db-conn [{:schema/version db-schema/version}])
+                  _ (swap! conns assoc db-name db-conn)
+                  stored (idb/get-item db-name)
+                  _ (if stored
+                      (let [stored-db (string->db stored)
+                            attached-db (d/db-with stored-db [(me-tx stored-db me)])]
+                        (reset-conn! db-conn attached-db)
+                        (when (not= (:schema stored-db) db-schema/schema) ;; check for code update
+                          (db-schema-changed-handler {:url repo})))
+                      (when logged?
+                        (d/transact! db-conn [(me-tx (d/db db-conn) me)])))
+                  _ (restore-config-handler repo)])))))))
 
 (defn- build-edges
   [edges]
@@ -2447,6 +2433,14 @@
             datoms (d/datoms filtered-db :eavt)]
         @(d/conn-from-datoms datoms db-schema/schema)))))
 
+(defn get-db-type
+  [repo]
+  (get-key-value repo :db/type))
+
+(defn local-native-fs?
+  [repo]
+  (= :local-native-fs (get-db-type repo)))
+
 ;; shortcut for query a block with string ref
 (defn qb
   [string-id]

src/main/frontend/db_schema.cljs

@@ -4,13 +4,16 @@
 
 (def files-db-schema
   {:file/path {:db/unique :db.unique/identity}
-   :file/content {}})
+   :file/content {}
+   :file/last-modified-at {}
+   :file/handle {}})
 
 ;; A page can corresponds to multiple files (same title),
 ;; a month journal file can have multiple pages,
 ;; also, each block can be treated as a page too.
 (def schema
-  {:schema/version {}
+  {:schema/version  {}
+   :db/type         {}
    :db/ident        {:db/unique :db.unique/identity}
 
    ;; user

src/main/frontend/dicts.cljs

@@ -67,13 +67,13 @@ title: How to take dummy notes?
 
 ## Hello, I'm a block!
 :PROPERTIES:
-:custom_id: 5f713e91-8a3c-4b04-a33a-c39482428e2d
+:id: 5f713e91-8a3c-4b04-a33a-c39482428e2d
 :END:
 ### I'm a child block!
 ### I'm another child block!
 ## Hey, I'm another block!
 :PROPERTIES:
-:custom_id: 5f713ea8-8cba-403d-ac00-9964b1ec7190
+:id: 5f713ea8-8cba-403d-ac00-9964b1ec7190
 :END:
 "
         :on-boarding/title "Hi, welcome to Logseq!"

src/main/frontend/fs.cljs

@@ -1,10 +1,53 @@
 (ns frontend.fs
-  (:require [frontend.util :as util]))
+  (:require [frontend.util :as util]
+            [frontend.config :as config]
+            [clojure.string :as string]
+            [frontend.idb :as idb]
+            [promesa.core :as p]
+            ["/frontend/utils" :as utils]))
+
+;; TODO:
+;; We need to support several platforms:
+;; 1. Chrome native file system API (lighting-fs wip)
+;; 2. IndexedDB (lighting-fs)
+;; 3. NodeJS
+#_(defprotocol Fs
+    (mkdir! [this dir])
+    (readdir! [this dir])
+    (unlink! [this path opts])
+    (rename! [this old-path new-path])
+    (rmdir! [this dir])
+    (read-file [dir path option])
+    (write-file! [dir path content])
+    (stat [dir path]))
+
+(defn local-db?
+  [dir]
+  (and (string? dir)
+       (config/local-db? (subs dir 1))))
 
 (defn mkdir
   [dir]
-  (when (and dir js/window.pfs)
-    (js/window.pfs.mkdir dir)))
+  (cond
+    (local-db? dir)
+    (let [[root new-dir] (rest (string/split dir "/"))
+          root-handle (str "handle-" root)]
+      (p/let [handle (idb/get-item root-handle)]
+        (when (and handle new-dir
+                   (not (string/blank? new-dir)))
+          (-> (p/let [handle (.getDirectoryHandle ^js handle new-dir
+                                                  #js {:create true})
+                      _ (idb/set-item! (str root-handle "/" new-dir) handle)]
+                (println "Stored handle: " (str root-handle "/" new-dir)))
+              (p/catch (fn [error]
+                         (println "mkdir error: " error)
+                         (js/console.error error)))))))
+
+    (and dir js/window.pfs)
+    (js/window.pfs.mkdir dir)
+
+    :else
+    (println (str "mkdir " dir " failed"))))
 
 (defn readdir
   [dir]
@@ -20,22 +63,50 @@
   (js/window.pfs.rename old-path new-path))
 
 (defn rmdir
+  "Remove the directory recursively."
   [dir]
   (js/window.workerThread.rimraf dir))
 
 (defn read-file
-  [dir path]
-  (js/window.pfs.readFile (str dir "/" path)
-                          (clj->js {:encoding "utf8"})))
-
-(defn read-file-2
-  [dir path]
-  (js/window.pfs.readFile (str dir "/" path)
-                          (clj->js {})))
+  ([dir path]
+   (read-file dir path (clj->js {:encoding "utf8"})))
+  ([dir path option]
+   (js/window.pfs.readFile (str dir "/" path) option)))
 
 (defn write-file
   [dir path content]
-  (and js/window.pfs (js/window.pfs.writeFile (str dir "/" path) content)))
+  (cond
+    (local-db? dir)
+    (let [parts (string/split path "/")
+          basename (last parts)
+          sub-dir (->> (butlast parts)
+                       (remove string/blank?)
+                       (string/join "/"))
+          handle-path (str "handle-"
+                           (subs dir 1)
+                           (if sub-dir
+                             (str "/" sub-dir)))
+          basename-handle-path (str handle-path "/" basename)]
+      (p/let [file-handle (idb/get-item basename-handle-path)]
+        (if file-handle
+          (utils/writeFile file-handle content)
+          ;; create file handle
+          (->
+           (p/let [handle (idb/get-item handle-path)]
+             (if handle
+               (p/let [file-handle (.getFileHandle ^js handle basename #js {:create true})
+                       _ (idb/set-item! basename-handle-path file-handle)]
+                 (utils/writeFile file-handle content))
+               (println "Error: directory handle not exists: " handle-path)))
+           (p/catch (fn [error]
+                      (println "Write local file failed: " {:path path})
+                      (js/console.error error)))))))
+
+    js/window.pfs
+    (js/window.pfs.writeFile (str dir "/" path) content)
+
+    :else
+    nil))
 
 (defn stat
   [dir path]

src/main/frontend/handler.cljs

@@ -15,6 +15,7 @@
             [frontend.handler.file :as file-handler]
             [frontend.handler.ui :as ui-handler]
             [frontend.handler.export :as export-handler]
+            [frontend.handler.web.nfs :as nfs]
             [frontend.ui :as ui]
             [goog.object :as gobj]
             [frontend.helper :as helper]
@@ -177,6 +178,7 @@
        (notification/show! "Sorry, it seems that your browser doesn't support IndexedDB, we recommend to use latest Chrome(Chromium) or Firefox(Non-private mode)." :error false)
        (state/set-indexedb-support! false)))
 
+    (nfs/trigger-check!)
     (restore-and-setup! me repos logged?)
 
     (periodically-persist-repo-to-indexeddb!)

src/main/frontend/handler/common.cljs

@@ -18,8 +18,6 @@
     ;; TODO: what if the remote is not named "origin", check the api from isomorphic-git
     (git/resolve-ref repo-url (str "refs/remotes/origin/" branch))))
 
-
-;; Should include un-pushed committed files too
 (defn check-changed-files-status
   ([]
    (check-changed-files-status (state/get-current-repo)))

src/main/frontend/handler/git.cljs

@@ -10,8 +10,9 @@
             [frontend.handler.notification :as notification]
             [frontend.handler.route :as route-handler]
             [frontend.handler.common :as common-handler]
-            [cljs-time.local :as tl]
-            [frontend.helper :as helper]))
+            [frontend.helper :as helper]
+            [frontend.config :as config]
+            [cljs-time.local :as tl]))
 
 (defn- set-git-status!
   [repo-url value]
@@ -31,12 +32,13 @@
   ([repo-url file]
    (git-add repo-url file true))
   ([repo-url file update-status?]
-   (-> (p/let [result (git/add repo-url file)]
-         (when update-status?
-           (common-handler/check-changed-files-status)))
-       (p/catch (fn [error]
-                  (println "git add '" file "' failed: " error)
-                  (js/console.error error))))))
+   (when-not (config/local-db? repo-url)
+     (-> (p/let [result (git/add repo-url file)]
+           (when update-status?
+             (common-handler/check-changed-files-status)))
+         (p/catch (fn [error]
+                    (println "git add '" file "' failed: " error)
+                    (js/console.error error)))))))
 
 (defn commit-and-force-push!
   [commit-message pushing?]

src/main/frontend/handler/image.cljs

@@ -33,8 +33,9 @@
                      (subs path 1)
                      path)]
           (util/p-handle
-           (fs/read-file-2 (util/get-repo-dir (state/get-current-repo))
-                           path)
+           (fs/read-file (util/get-repo-dir (state/get-current-repo))
+                           path
+                           {})
            (fn [blob]
              (let [blob (js/Blob. (array blob) (clj->js {:type "image"}))
                    img-url (image/create-object-url blob)]

src/main/frontend/handler/repo.cljs

@@ -62,33 +62,23 @@
         (p/let [file-exists? (fs/create-if-not-exists repo-dir (str app-dir "/" config/config-file) default-content)]
           (let [path (str app-dir "/" config/config-file)
                 old-content (when file-exists?
-                              (db/get-file repo-url path))
-                content (or
-                         (and old-content
-                              (string/replace old-content "heading" "block"))
-                         default-content)]
-            (db/reset-file! repo-url path content)
-            (db/reset-config! repo-url content)
-            (when-not (= content old-content)
-              (git-handler/git-add repo-url path))))
-        ;; (p/let [file-exists? (fs/create-if-not-exists repo-dir (str app-dir "/" config/metadata-file) default-content)]
-        ;;   (let [path (str app-dir "/" config/metadata-file)]
-        ;;     (when-not file-exists?
-        ;;       (db/reset-file! repo-url path "{:tx-data []}")
-        ;;       (git-handler/git-add repo-url path))))
-))))
+                              (db/get-file repo-url path))]
+            (db/reset-file! repo-url path default-content)
+            (db/reset-config! repo-url default-content)
+            (when (not= default-content old-content)
+              (git-handler/git-add repo-url path))))))))
 
 (defn create-contents-file
   [repo-url]
   (spec/validate :repos/url repo-url)
   (let [repo-dir (util/get-repo-dir repo-url)
         format (state/get-preferred-format)
-        path (str "pages/contents." (if (= (name format) "markdown")
-                                      "md"
-                                      (name format)))
+        path (str (state/get-pages-directory)
+                  "/contents."
+                  (if (= (name format) "markdown") "md" (name format)))
         file-path (str "/" path)
         default-content (util/default-content-with-title format "contents")]
-    (p/let [_ (-> (fs/mkdir (str repo-dir "/pages"))
+    (p/let [_ (-> (fs/mkdir (str repo-dir "/" (state/get-pages-directory)))
                   (p/catch (fn [_e])))
             file-exists? (fs/create-if-not-exists repo-dir file-path default-content)]
       (when-not file-exists?
@@ -167,43 +157,54 @@
     (create-contents-file repo-url)
     (create-custom-theme repo-url)))
 
+(defn- parse-files-and-load-to-db!
+  [repo-url files contents {:keys [first-clone? delete-files delete-blocks re-render? additional-files-info]}]
+  (state/set-state! :repo/loading-files? false)
+  (state/set-state! :repo/importing-to-db? true)
+  (let [parsed-files (filter
+                      (fn [[file _]]
+                        (let [format (format/get-format file)]
+                          (contains? config/mldoc-support-formats format)))
+                      contents)
+        blocks-pages (if (seq parsed-files)
+                       (db/extract-all-blocks-pages repo-url parsed-files)
+                       [])]
+    (db/reset-contents-and-blocks! repo-url contents blocks-pages delete-files delete-blocks)
+    (let [config-file (str config/app-name "/" config/config-file)]
+      (if (contains? (set files) config-file)
+        (when-let [content (get contents config-file)]
+          (file-handler/restore-config! repo-url content true))))
+    (when first-clone? (create-default-files! repo-url))
+    (state/set-state! :repo/importing-to-db? false)
+    (when re-render?
+      (ui-handler/re-render-root!))))
+
 (defn load-repo-to-db!
-  [repo-url diffs first-clone?]
+  [repo-url {:keys [first-clone? diffs nfs-files nfs-contents additional-files-info]}]
   (spec/validate :repos/url repo-url)
-  (let [load-contents (fn [files delete-files delete-blocks re-render?]
+  (let [load-contents (fn [files option]
                         (file-handler/load-files-contents!
                          repo-url
                          files
-                         (fn [contents]
-                           (state/set-state! :repo/loading-files? false)
-                           (state/set-state! :repo/importing-to-db? true)
-                           (let [parsed-files (filter
-                                               (fn [[file _]]
-                                                 (let [format (format/get-format file)]
-                                                   (contains? config/mldoc-support-formats format)))
-                                               contents)
-                                 blocks-pages (if (seq parsed-files)
-                                                (db/extract-all-blocks-pages repo-url parsed-files)
-                                                [])]
-                             (db/reset-contents-and-blocks! repo-url contents blocks-pages delete-files delete-blocks)
-                             (let [config-file (str config/app-name "/" config/config-file)]
-                               (if (contains? (set files) config-file)
-                                 (when-let [content (get contents config-file)]
-                                   (file-handler/restore-config! repo-url content true))))
-                             (when first-clone? (create-default-files! repo-url))
-                             (state/set-state! :repo/importing-to-db? false)
-                             (when re-render?
-                               (ui-handler/re-render-root!))))))]
-    (if first-clone?
+                         (fn [contents] (parse-files-and-load-to-db! repo-url files contents option))))]
+    (cond
+      (seq nfs-files)
+      (parse-files-and-load-to-db! repo-url nfs-files nfs-contents
+                                   {:first-clone? true
+                                    :additional-files-info additional-files-info})
+
+      first-clone?
       (->
        (p/let [files (file-handler/load-files repo-url)]
-         (load-contents files nil nil false))
+         (load-contents files {:first-clone? first-clone?}))
        (p/catch (fn [error]
                   (println "loading files failed: ")
                   (js/console.dir error)
                   ;; Empty repo
                   (create-default-files! repo-url)
                   (state/set-state! :repo/loading-files? false))))
+
+      :else
       (when (seq diffs)
         (let [filter-diffs (fn [type] (->> (filter (fn [f] (= type (:type f))) diffs)
                                            (map :path)))
@@ -217,7 +218,11 @@
                              (db/delete-pages-by-files remove-files)
                              [])
               add-or-modify-files (util/remove-nils (concat add-files modify-files))]
-          (load-contents add-or-modify-files (concat delete-files delete-pages) delete-blocks true))))))
+          (load-contents add-or-modify-files
+                         {:first-clone? first-clone?
+                          :delete-files (concat delete-files delete-pages)
+                          :delete-blocks delete-blocks
+                          :re-render? true}))))))
 
 (defn persist-repo!
   [repo]
@@ -231,7 +236,8 @@
   [repo-url diffs first-clone?]
   (spec/validate :repos/url repo-url)
   (when (or diffs first-clone?)
-    (load-repo-to-db! repo-url diffs first-clone?)))
+    (load-repo-to-db! repo-url {:first-clone? first-clone?
+                                :diffs diffs})))
 
 (defn transact-react-and-alter-file!
   [repo tx transact-option files]
@@ -460,6 +466,11 @@
                (fn [error]
                  (prn "Delete repo failed, error: " error))))
 
+(defn start-repo-db-if-not-exists!
+  [repo option]
+  (state/set-current-repo! repo)
+  (db/start-db-conn! nil repo option))
+
 (defn setup-local-repo-if-not-exists!
   []
   (if js/window.pfs

src/main/frontend/handler/web/nfs.cljs

@@ -0,0 +1,93 @@
+(ns frontend.handler.web.nfs
+  "The File System Access API, https://web.dev/file-system-access/."
+  (:require [cljs-bean.core :as bean]
+            [promesa.core :as p]
+            [goog.object :as gobj]
+            [goog.dom :as gdom]
+            [frontend.util :as util]
+            ["/frontend/utils" :as utils]
+            [frontend.handler.repo :as repo-handler]
+            [frontend.idb :as idb]
+            [frontend.state :as state]
+            [clojure.string :as string]
+            [frontend.ui :as ui]
+            [frontend.config :as config]))
+
+(defn ls-dir-files
+  []
+  (->
+   (p/let [result (utils/openDirectory #js {:recursive true})
+           root-handle (nth result 0)
+           dir-name (gobj/get root-handle "name")
+           repo (str config/local-db-prefix dir-name)
+           _ (idb/set-item! (str "handle-" repo) root-handle)
+           result (nth result 1)
+           result (flatten (bean/->clj result))
+           files (doall
+                  (map (fn [file]
+                         (let [handle (gobj/get file "handle")
+                               get-attr #(gobj/get file %)]
+                           {:file/path (get-attr "webkitRelativePath")
+                            :file/last-modified-at (get-attr "lastModified")
+                            :file/size (get-attr "size")
+                            :file/type (get-attr "type")
+                            :file/file file
+                            :file/handle handle})) result))
+           text-files (filter (fn [file] (contains? #{"org" "md" "markdown"} (util/get-file-ext (:file/path file)))) files)]
+     (doseq [file text-files]
+       (idb/set-item! (str "handle-" repo "/" (:file/path file))
+                      (:file/handle file)))
+     (-> (p/all (map (fn [file]
+                       (p/let [content (.text (:file/file file))]
+                         (assoc file :file/content content))) text-files))
+         (p/then (fn [result]
+                   (let [files (map #(dissoc % :file/file) result)]
+                     (repo-handler/start-repo-db-if-not-exists! repo {:db-type :local-native-fs})
+                     (repo-handler/load-repo-to-db! repo
+                                                    {:first-clone? true
+                                                     :nfs-files (map :file/path files)
+                                                     :nfs-contents (mapv (fn [f] [(:file/path f) (:file/content f)]) files)
+                                                     :additional-files-info files})
+                     ;; create default directories and files
+                     )))
+         (p/catch (fn [error]
+                    (println "Load files content error: ")
+                    (js/console.dir error)))))
+   (p/catch (fn [error]
+              (println "Open directory error: ")
+              (js/console.dir error)))))
+
+(defn open-file-picker
+  "Shows a file picker that lets a user select a single existing file, returning a handle for the selected file. "
+  ([]
+   (open-file-picker {}))
+  ([option]
+   (js/window.showOpenFilePicker (bean/->js option))))
+
+(defn get-local-repo
+  []
+  (when-let [repo (state/get-current-repo)]
+    (when (config/local-db? repo)
+      repo)))
+
+(defn check-directory-permission!
+  [repo]
+  (p/let [handle (idb/get-item (str "handle-" repo))]
+    (utils/verifyPermission handle true)))
+
+(defn ask-permission
+  [repo]
+  (fn [close-fn]
+    [:div
+     [:p.text-gray-700
+      "Grant native filesystem permission for directory: "
+      [:b (config/get-local-dir repo)]]
+     (ui/button
+       "Grant"
+       :on-click (fn []
+                   (check-directory-permission! repo)
+                   (close-fn)))]))
+
+(defn trigger-check! []
+  (when-let [repo (get-local-repo)]
+    (state/set-modal! (ask-permission repo))))

src/main/frontend/idb.cljs

@@ -0,0 +1,34 @@
+(ns frontend.idb
+  (:require ["localforage" :as localforage]
+            [cljs-bean.core :as bean]
+            [goog.object :as gobj]
+            [promesa.core :as p]))
+
+;; offline db
+(def store-name "dbs")
+(.config localforage
+         (bean/->js
+          {:name "logseq-datascript"
+           :version 1.0
+           :storeName store-name}))
+
+(defonce localforage-instance (.createInstance localforage store-name))
+
+(defn clear-store!
+  []
+  (p/let [_ (.clear localforage-instance)
+          dbs (js/window.indexedDB.databases)]
+    (doseq [db dbs]
+      (js/window.indexedDB.deleteDatabase (gobj/get db "name")))))
+
+(defn remove-item!
+  [key]
+  (.removeItem localforage-instance key))
+
+(defn set-item!
+  [key value]
+  (.setItem localforage-instance key value))
+
+(defn get-item
+  [key]
+  (.getItem localforage-instance key))

src/main/frontend/protocol/fs.cljs

@@ -0,0 +1,7 @@
+(ns frontend.protocol.fs)
+
+(defprotocol Fs
+  (load-directory! [this])
+  (write-file! [this path content])
+  (delete-file! [this path])
+  (get-file-stats! [this path]))

src/main/frontend/state.cljs

@@ -184,8 +184,10 @@
 
 (defn get-pages-directory
   []
-  (when-let [repo (get-current-repo)]
-    (:pages-directory (get-config repo))))
+  (or
+   (when-let [repo (get-current-repo)]
+     (:pages-directory (get-config repo)))
+   "pages"))
 
 (defn org-mode-file-link?
   [repo]

src/main/frontend/storage.cljs

@@ -2,7 +2,6 @@
   (:refer-clojure :exclude [get set remove])
   (:require [cljs.reader :as reader]))
 
-;; TODO: deprecate this, will persistent datascript
 (defn get
   [key]
   (reader/read-string ^js (.getItem js/localStorage (name key))))

src/main/frontend/utils.js

@@ -71,4 +71,68 @@ export var getSelectionText = function() {
   }
 
   return '';
+
+// Modified from https://github.com/GoogleChromeLabs/browser-nativefs
+// because shadow-cljs doesn't handle this babel transform
+const getFiles = async function (dirHandle, recursive) {
+  const dirs = [];
+  const files = [];
+  const path = dirHandle.name;
+  for await (const entry of dirHandle.values()) {
+    const nestedPath = `${path}/${entry.name}`;
+    if (entry.kind === 'file') {
+      files.push(
+        entry.getFile().then((file) => {
+          Object.defineProperty(file, 'webkitRelativePath', {
+            configurable: true,
+            enumerable: true,
+            get: () => nestedPath,
+          });
+          Object.defineProperty(file, 'handle', {
+            configurable: true,
+            enumerable: true,
+            get: () => entry,
+          });
+          return file;
+        }
+        )
+      );
+    } else if (entry.kind === 'directory' && recursive) {
+      dirs.push(getFiles(entry, recursive, nestedPath));
+    }
+  }
+
+  return [(await Promise.all(dirs)), (await Promise.all(files))];
+};
+
+export var openDirectory = async function (options = {}) {
+  options.recursive = options.recursive || false;
+  const handle = await window.showDirectoryPicker({ mode: 'readwrite' });
+  return [handle, getFiles(handle, options.recursive)];
+};
+
+export var writeFile = async function (fileHandle, contents) {
+  // Create a FileSystemWritableFileStream to write to.
+  const writable = await fileHandle.createWritable();
+  // Write the contents of the file to the stream.
+  await writable.write(contents);
+  // Close the file and write the contents to disk.
+  await writable.close();
+};
+
+export var verifyPermission = async function (handle, readWrite) {
+  const options = {};
+  if (readWrite) {
+    options.mode = 'readwrite';
+  }
+  // Check if permission was already granted. If so, return true.
+  if ((await handle.queryPermission(options)) === 'granted') {
+    return true;
+  }
+  // Request permission. If the user grants permission, return true.
+  if ((await handle.requestPermission(options)) === 'granted') {
+    return true;
+  }
+  // The user didn't grant permission, so return false.
+  return false;
 }