Both load && persist work now

src/main/frontend/db/persist.cljs

@@ -14,7 +14,7 @@
   (p/let [repos (idb/get-nfs-dbs)
           db-repos (persist-db/<list-db)
           electron-disk-graphs (when (util/electron?) (ipc/ipc "getGraphs"))]
-    (distinct (concat repos db-repos (bean/->clj electron-disk-graphs)))))
+    (distinct (concat repos db-repos (some-> electron-disk-graphs bean/->clj)))))
 
 (defn get-serialized-graph
   [graph-name]

src/main/frontend/db_worker.cljs

@@ -14,6 +14,16 @@
 (defonce *sqlite (atom nil))
 (defonce *sqlite-db (atom nil))
 (defonce *datascript-conn (atom nil))
+(defonce *opfs-pool (atom nil))
+
+(defn- get-opfs-pool
+  []
+  (or @*opfs-pool
+      (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name "logseq-db"
+                                                         :initialCapacity 100})]
+        ;; (.removeVfs pool)
+        (reset! *opfs-pool pool)
+        pool)))
 
 (defn- init-sqlite-module!
   []
@@ -23,11 +33,25 @@
             sqlite (sqlite3InitModule (clj->js {:url sqlite-wasm-url
                                                 :print js/console.log
                                                 :printErr js/console.error}))]
-      (reset! *sqlite sqlite))))
+      (reset! *sqlite sqlite)
+      nil)))
+
+(defn- remove-pfs!
+  "!! use it only for development"
+  []
+  (when-let [^js pool (get-opfs-pool)]
+    (.removeVfs ^js pool)))
 
-(defn- close-all-dbs!
+(defn- get-file-names
   []
-  )
+  (when-let [^js pool (get-opfs-pool)]
+    (.getFileNames pool)))
+
+(defn- export-db-file
+  [file-path]
+  ;; TODO: get file name by repo
+  (when-let [^js pool (get-opfs-pool)]
+    (.exportFile ^js pool file-path)))
 
 (defn upsert-addr-content!
   "Upsert addr+data-seq"
@@ -62,8 +86,21 @@
           (upsert-addr-content! data)))
 
       (-restore [_ addr]
-        (let [content (restore-data-from-addr addr)]
-          (edn/read-string content))))))
+        (restore-data-from-addr addr)))))
+
+(defn- create-or-open-db!
+  [repo]
+  (p/let [pool (get-opfs-pool)
+          db (new (.-OpfsSAHPoolDb pool) (str "/" repo ".sqlite"))
+          storage (new-sqlite-storage repo {})]
+    (js/console.dir db)
+    (reset! *sqlite-db db)
+    (.exec db "PRAGMA locking_mode=exclusive")
+    (.exec db "create table if not exists kvs (addr INTEGER primary key, content TEXT)")
+    (let [conn (or (d/restore-conn storage)
+                   (d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
+      (reset! *datascript-conn conn)
+      nil)))
 
 #_:clj-kondo/ignore
 (defclass SQLiteDB
@@ -90,8 +127,7 @@
 
   (init
    [_this]
-   (init-sqlite-module!)
-   nil)
+   (init-sqlite-module!))
 
   (inited
    [_this]
@@ -99,24 +135,14 @@
 
   (listDB
    [_this]
-   ;; (.list_db sqlite-db)
-   #js [])
+   ;; TODO:
+   (prn (get-file-names))
+   nil)
 
-  (newDB
+  (createOrOpenDB
    [_this repo]
    ;; TODO: close all the other db connections
-   (p/let [sqlite @*sqlite
-           db-name repo
-           pool (.installOpfsSAHPoolVfs sqlite #js {:name db-name})
-           db (new (.-OpfsSAHPoolDb pool) "/logseq")
-           storage (new-sqlite-storage db-name {})]
-     (reset! *sqlite-db db)
-     (.exec db "PRAGMA locking_mode=exclusive")
-     (.exec db "create table if not exists kvs (addr INTEGER primary key, content TEXT)")
-     (let [conn (or (d/restore-conn storage)
-                    (d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
-       (reset! *datascript-conn conn)
-       nil)))
+   (create-or-open-db! repo))
 
   (transact
    [_this repo tx-data tx-meta]
@@ -142,5 +168,4 @@
   "web worker entry"
   []
   (let [^js obj (SQLiteDB.)]
-    (p/let [_ (init-sqlite-module!)]
-      (Comlink/expose obj))))
+    (Comlink/expose obj)))

src/main/frontend/handler.cljs

@@ -75,58 +75,52 @@
     (f)
     (js/setInterval f 5000)))
 
-(defn- instrument!
-  []
-  (let [total (srs/get-srs-cards-total)]
-    (state/set-state! :srs/cards-due-count total)))
-
 (defn restore-and-setup!
-  [repos]
-  (when-let [repo (or (state/get-current-repo) (:url (first repos)))]
-    (-> (p/do!
-         (db-restore/restore-graph! repo)
-         (repo-config-handler/start {:repo repo})
-         (op-mem-layer/<init-load-from-indexeddb! repo))
-        (p/then
-         (fn []
-           (db-listener/listen-and-persist! repo)
+  [repo repos]
+  (-> (p/do!
+       (db-restore/restore-graph! repo)
+       (repo-config-handler/start {:repo repo})
+       (op-mem-layer/<init-load-from-indexeddb! repo))
+      (p/then
+       (fn []
+         (db-listener/listen-and-persist! repo)
            ;; try to load custom css only for current repo
-           (ui-handler/add-style-if-exists!)
-
-           (->
-            (p/do!
-             (when (config/global-config-enabled?)
-               (global-config-handler/start {:repo repo}))
-             (when (config/plugin-config-enabled?)
-               (plugin-config-handler/start)))
-            (p/finally
-              (fn []
+         (ui-handler/add-style-if-exists!)
+
+         (->
+          (p/do!
+           (when (config/global-config-enabled?)
+             (global-config-handler/start {:repo repo}))
+           (when (config/plugin-config-enabled?)
+             (plugin-config-handler/start)))
+          (p/finally
+            (fn []
                 ;; install after config is restored
-                (shortcut/refresh!)
+              (shortcut/refresh!)
 
-                (cond
-                  (and (not (seq (db/get-files config/local-repo)))
+              (cond
+                (and (not (seq (db/get-files config/local-repo)))
                        ;; Not native local directory
-                       (not (some config/local-file-based-graph? (map :url repos)))
-                       (not (mobile-util/native-platform?)))
+                     (not (some config/local-file-based-graph? (map :url repos)))
+                     (not (mobile-util/native-platform?)))
                   ;; will execute `(state/set-db-restoring! false)` inside
-                  (repo-handler/setup-local-repo-if-not-exists!)
+                (repo-handler/setup-local-repo-if-not-exists!)
 
-                  :else
-                  (state/set-db-restoring! false)))))))
-        (p/then
-         (fn []
-           (js/console.log "db restored, setting up repo hooks")
+                :else
+                (state/set-db-restoring! false)))))))
+      (p/then
+       (fn []
+         (js/console.log "db restored, setting up repo hooks")
 
-           (state/pub-event! [:modal/nfs-ask-permission])
+         (state/pub-event! [:modal/nfs-ask-permission])
 
-           (page-handler/init-commands!)
+         (page-handler/init-commands!)
 
-           (watch-for-date!)
-           (file-handler/watch-for-current-graph-dir!)
-           (state/pub-event! [:graph/restored (state/get-current-repo)])))
-        (p/catch (fn [error]
-                   (log/error :exception error))))))
+         (watch-for-date!)
+         (file-handler/watch-for-current-graph-dir!)
+         (state/pub-event! [:graph/restored (state/get-current-repo)])))
+      (p/catch (fn [error]
+                 (log/error :exception error)))))
 
 (defn- handle-connection-change
   [e]
@@ -169,27 +163,30 @@
 ;; FIXME: Another get-repos implementation at src\main\frontend\handler\repo.cljs
 (defn- get-repos
   []
-  (p/let [nfs-dbs (db-persist/get-all-graphs)]
-    ;; TODO: Better IndexDB migration handling
-    (cond
-      (and (mobile-util/native-platform?)
-           (some #(or (string/includes? % " ")
-                      (string/includes? % "logseq_local_/")) nfs-dbs))
-      (do (notification/show! ["DB version is not compatible, please clear cache then re-add your graph back."
-                               (ui/button
-                                (t :settings-page/clear-cache)
-                                :class    "ui__modal-enter"
-                                :class    "text-sm p-1"
-                                :on-click clear-cache!)] :error false)
-          {:url config/local-repo
-           :example? true})
-
-      (seq nfs-dbs)
-      (map (fn [db] {:url db :nfs? true}) nfs-dbs)
-
-      :else
-      [{:url config/local-repo
-        :example? true}])))
+  (->
+   (p/let [nfs-dbs (db-persist/get-all-graphs)]
+     ;; TODO: Better IndexDB migration handling
+     (cond
+       (and (mobile-util/native-platform?)
+            (some #(or (string/includes? % " ")
+                       (string/includes? % "logseq_local_/")) nfs-dbs))
+       (do (notification/show! ["DB version is not compatible, please clear cache then re-add your graph back."
+                                (ui/button
+                                 (t :settings-page/clear-cache)
+                                 :class    "ui__modal-enter"
+                                 :class    "text-sm p-1"
+                                 :on-click clear-cache!)] :error false)
+           {:url config/local-repo
+            :example? true})
+
+       (seq nfs-dbs)
+       (map (fn [db] {:url db :nfs? true}) nfs-dbs)
+
+       :else
+       [{:url config/local-repo
+         :example? true}]))
+   (p/catch (fn [error]
+              (js/console.error)))))
 
 (defn- register-components-fns!
   []
@@ -214,7 +211,6 @@
   [render]
   (test/setup-test!)
   (get-system-info)
-  (db-browser/start-db-worker!)
   (set-global-error-notification!)
 
   (set! js/window.onhashchange #(state/hide-custom-context-menu!)) ;; close context menu when page navs
@@ -244,10 +240,12 @@
   (p/do!
    (when (mobile-util/native-platform?)
      (mobile/mobile-preinit))
-   (-> (p/let [repos (get-repos)
+   (-> (p/let [_ (db-browser/start-db-worker!)
+               repos (get-repos)
                _ (state/set-repos! repos)
                _ (mobile-util/hide-splash) ;; hide splash as early as ui is stable
-               _ (restore-and-setup! repos)]
+               repo (or (state/get-current-repo) (:url (first repos)))
+               _ (restore-and-setup! repo repos)]
          (when (mobile-util/native-platform?)
            (state/restore-mobile-theme!)))
        (p/catch (fn [e]
@@ -260,8 +258,7 @@
 
    (when config/dev?
      (enable-datalog-console))
-   (persist-var/load-vars)
-   (js/setTimeout instrument! (* 60 1000))))
+   (persist-var/load-vars)))
 
 (defn stop! []
   (prn "stop!"))

src/main/frontend/persist_db.cljs

@@ -29,6 +29,6 @@
   ([repo]
    (<fetch-init-data repo {}))
   ([repo opts]
-   (p/let [ret (protocol/<fetch-initital-data (get-impl) repo opts)]
-     (js/console.log "fetch-initital" ret)
+   (p/let [ret (protocol/<fetch-initial-data (get-impl) repo opts)]
+     (js/console.log "fetch-initial-data" ret)
      ret)))

src/main/frontend/persist_db/browser.cljs

@@ -8,7 +8,8 @@
             [frontend.config :as config]
             [promesa.core :as p]
             [frontend.util :as util]
-            [frontend.handler.notification :as notification]))
+            [frontend.handler.notification :as notification]
+            [cljs-bean.core :as bean]))
 
 (defonce *sqlite (atom nil))
 
@@ -20,16 +21,16 @@
                        "/static/js/db-worker.js")
           worker (js/Worker. worker-url)
           sqlite (Comlink/wrap worker)]
-      (reset! *sqlite sqlite))))
+      (reset! *sqlite sqlite)
+      (.init sqlite))))
 
 (defrecord InBrowser []
   protocol/PersistentDB
   (<new [_this repo]
-    (prn :debug ::new-repo repo)
     (let [^js sqlite @*sqlite]
-      (-> (.newDB sqlite repo)
-          (p/then (fn [result]
-                    (prn "SQLite db created successfully: " repo)))
+      (-> (.createOrOpenDB sqlite repo)
+          (p/then (fn [_result]
+                    (prn "SQLite db created or opened successfully: " repo)))
           (p/catch (fn [error]
                      (js/console.error error)
                      (notification/show! [:div (str "SQLiteDB creation error: " error)] :error)
@@ -38,6 +39,8 @@
   (<list-db [_this]
     (when-let [^js sqlite @*sqlite]
       (-> (.listDB sqlite)
+          (p/then (fn [result]
+                    (bean/->clj result)))
           (p/catch (fn [error]
                      (prn :debug :list-db-error (js/Date.))
                      (notification/show! [:div (str "SQLiteDB error: " error)] :error)
@@ -50,21 +53,16 @@
     )
 
   (<transact-data [_this repo tx-data tx-meta]
-    (prn :debug ::transact-data repo (count tx-data) (count tx-meta))
-    (let [^js sqlite @*sqlite]
+    (when-let [^js sqlite @*sqlite]
       (p->c
        (p/let [_ (.transact sqlite repo (pr-str tx-data) (pr-str tx-meta))]
          nil))))
 
-  (<fetch-initital-data [_this repo _opts]
-    (prn ::fetch-initital-data repo @*sqlite)
-    (-> (let [^js sqlite @*sqlite
-                ;; <fetch-initital-data is called when init/re-loading graph
-                ;; the underlying DB should be opened
-              ]
-          (p/let [_ (.newDB sqlite repo)]
-            (.getInitialData sqlite repo)))
-        (p/catch (fn [error]
-                   (prn :debug :fetch-initial-data-error)
-                   (js/console.error error)
-                   (notification/show! [:div (str "SQLiteDB fetch error: " error)] :error) {})))))
+  (<fetch-initial-data [_this repo _opts]
+    (when-let [^js sqlite @*sqlite]
+      (-> (p/let [_ (.createOrOpenDB sqlite repo)]
+            (.getInitialData sqlite repo))
+          (p/catch (fn [error]
+                     (prn :debug :fetch-initial-data-error)
+                     (js/console.error error)
+                     (notification/show! [:div (str "SQLiteDB fetch error: " error)] :error) {}))))))

src/main/frontend/persist_db/protocol.cljs

@@ -8,4 +8,4 @@
   (<new [this repo])
   (<unsafe-delete [this repo])
   (<transact-data [this repo tx-data tx-meta] "Transact data to db")
-  (<fetch-initital-data [this repo opts]))
+  (<fetch-initial-data [this repo opts]))