wip: replace sqlite-db with official sqlite-wasm

src/main/frontend/db_worker.cljs

@@ -1,63 +1,70 @@
 (ns frontend.db-worker
   "Worker used for browser DB implementation"
-  (:require ["@logseq/sqlite" :as sqlite-db :default wasm-bindgen-init]
-            ["comlink" :as Comlink]
-            [promesa.core :as p]
+  (:require [promesa.core :as p]
             [datascript.storage :refer [IStorage]]
             [cljs.cache :as cache]
-            [cljs.reader :as reader]
+            [clojure.edn :as edn]
             [datascript.core :as d]
             [logseq.db.frontend.schema :as db-schema]
             [shadow.cljs.modern :refer [defclass]]
             [datascript.transit :as dt]
-            [clojure.edn :as edn]
-            [clojure.string :as string]
-            ["@logseq/sqlite-wasm" :as sqlite-wasm]))
-
-(def *wasm-loaded (atom false))
+            ["@logseq/sqlite-wasm" :default sqlite3InitModule]
+            ["comlink" :as Comlink]))
 
-;; datascript conns
-(defonce conns (atom nil))
+(defonce *sqlite (atom nil))
+(defonce *sqlite-db (atom nil))
+(defonce *datascript-conn (atom nil))
 
-(defn get-conn
-  [repo]
-  (get @conns repo))
+(defn- init-sqlite-module!
+  []
+  (when-not @*sqlite
+    (p/let [base-url (str js/self.location.protocol "//" js/self.location.host)
+            sqlite-wasm-url (str base-url "/js/")
+            sqlite (sqlite3InitModule (clj->js {:url sqlite-wasm-url
+                                                :print js/console.log
+                                                :printErr js/console.error}))]
+      (reset! *sqlite sqlite))))
+
+(defn- close-all-dbs!
+  []
+  )
 
 (defn upsert-addr-content!
   "Upsert addr+data-seq"
-  [repo data]
-  (.upsert_addr_content sqlite-db repo data))
+  [data]
+  (assert (some? @*sqlite-db) "sqlite db not exists")
+  (let [^Object db @*sqlite-db]
+    (.transaction db (fn [tx]
+                       (doseq [item data]
+                         (.exec tx #js {:sql "INSERT INTO kvs (addr, content) values ($addr, $content) on conflict(addr) do update set content = $content"
+                                        :bind item}))))))
 
 (defn restore-data-from-addr
-  [repo addr]
-  (.get_content_by_addr sqlite-db repo addr))
-
+  [addr]
+  (assert (some? @*sqlite-db) "sqlite db not exists")
+  (when-let [content (-> (.exec @*sqlite-db #js {:sql "select content from kvs where addr = ?"
+                                                 :bind #js [addr]
+                                                 :rowMode "array"})
+                         ffirst)]
+    (edn/read-string content)))
 
 (defn new-sqlite-storage
-  [repo {:keys [threshold]
-         :or {threshold 4096}}]
+  [_repo {:keys [threshold]
+          :or {threshold 4096}}]
   (let [_cache (cache/lru-cache-factory {} :threshold threshold)]
     (reify IStorage
       (-store [_ addr+data-seq]
-        (let [data (->>
-                    (map
-                     (fn [[addr data]]
-                       #js {:addr addr
-                            :content (pr-str data)})
-                     addr+data-seq)
-                    (to-array))]
-          (upsert-addr-content! repo data)
-          {:result "ok"}))
+        (let [data (map
+                    (fn [[addr data]]
+                      #js {:$addr addr
+                           :$content (pr-str data)})
+                    addr+data-seq)]
+          (upsert-addr-content! data)))
 
       (-restore [_ addr]
-        (let [content (restore-data-from-addr repo addr)]
+        (let [content (restore-data-from-addr addr)]
           (edn/read-string content))))))
 
-(defn split-last [pattern s]
-  (when-let [last-index (string/last-index-of s pattern)]
-    [(subs s 0 last-index)
-     (subs s (+ last-index (count pattern)) (count s))]))
-
 #_:clj-kondo/ignore
 (defclass SQLiteDB
   (extends js/Object)
@@ -67,121 +74,69 @@
    (super))
 
   Object
-  (init
-   [_this]
-   (let [[_ sqlite-wasm-url] (split-last "url=" (.. js/location -href))]
-     (assert (some? sqlite-wasm-url) "sqlite-wasm-url is empty")
-     (p/let [wasm-url (js/URL. sqlite-wasm-url (.. js/location -href))
-            _ (wasm-bindgen-init wasm-url)]
-      (prn ::init-ok
-           :has-opfs-support (.has_opfs_support sqlite-db)
-           :sqlite-version (.get_version sqlite-db))
-      (reset! *wasm-loaded true))))
 
-  (inited
-   [_this]
-   (boolean @*wasm-loaded))
+  ;; ;; dev-only, close all db connections and db files
+  ;; (unsafeDevCloseAll
+  ;;  [_this]
+  ;;  (.dev_close sqlite-db))
 
-  ;; dev-only, close all db connections and db files
-  (unsafeDevCloseAll
-   [_this]
-   (.dev_close sqlite-db))
+  ;; (getVersion
+  ;;  [_this]
+  ;;  (.get_version sqlite-db))
 
-  (getVersion
+  ;; (supportOPFS
+  ;;  [_this]
+  ;;  (.has_opfs_support sqlite-db))
+
+  (init
    [_this]
-   (.get_version sqlite-db))
+   (init-sqlite-module!)
+   nil)
 
-  (supportOPFS
+  (inited
    [_this]
-   (.has_opfs_support sqlite-db))
+   (some? @*sqlite))
 
   (listDB
    [_this]
-   (.list_db sqlite-db))
+   ;; (.list_db sqlite-db)
+   #js [])
 
   (newDB
    [_this repo]
-   (p/do!
-    (.ensure_init sqlite-db)
-    (.init_db sqlite-db repo) ;; close another and init this one
-    (.new_db sqlite-db repo)
-    (let [db-name repo
-          storage (new-sqlite-storage db-name {})
-          conn (or (d/restore-conn storage)
-                   (d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
-      (swap! conns assoc db-name conn)
-      nil)))
+   ;; TODO: close all the other db connections
+   (p/let [sqlite @*sqlite
+           db-name repo
+           pool (.installOpfsSAHPoolVfs sqlite #js {:name db-name})
+           db (new (.-OpfsSAHPoolDb pool) "/logseq")
+           storage (new-sqlite-storage db-name {})]
+     (reset! *sqlite-db db)
+     (.exec db "PRAGMA locking_mode=exclusive")
+     (.exec db "create table if not exists kvs (addr INTEGER primary key, content TEXT)")
+     (let [conn (or (d/restore-conn storage)
+                    (d/create-conn db-schema/schema-for-db-based-graph {:storage storage}))]
+       (reset! *datascript-conn conn)
+       nil)))
 
   (transact
    [_this repo tx-data tx-meta]
-   (when-let [conn (get-conn repo)]
+   (when-let [conn @*datascript-conn]
      (try
-       (let [tx-data (reader/read-string tx-data)
-             tx-meta (reader/read-string tx-meta)]
-         (d/transact! conn tx-data tx-meta))
+       (let [tx-data (edn/read-string tx-data)
+             tx-meta (edn/read-string tx-meta)]
+         (d/transact! conn tx-data tx-meta)
+         nil)
        (catch :default e
          (prn :debug :error)
          (js/console.error e)))))
 
   (getInitialData
    [_this repo]
-   (when-let [conn (get-conn repo)]
+   (when-let [conn @*datascript-conn]
      (let [db @conn]
        (->> (d/datoms db :eavt)
-              ;; (remove (fn [e] (= :block/content (:a e))))
             vec
-            dt/write-transit-str))))
-
-  (openDB
-   [_this repo]
-   (p/do!
-    (.ensure_init sqlite-db)
-    ;; close another and init this one
-    (.init_db sqlite-db repo)))
-
-  (deleteBlocks
-   [_this repo uuids]
-   (when (seq uuids)
-     (p/do!
-      (.ensure_init sqlite-db)
-      (.delete_blocks sqlite-db repo uuids))))
-
-  (upsertBlocks
-   [_this repo blocks]
-   (p/do!
-    (.ensure_init sqlite-db)
-    (.upsert_blocks sqlite-db repo blocks)))
-
-  (fetchAllPages
-   [_this repo]
-   (p/do!
-    (.ensure_init sqlite-db)
-    (.fetch_all_pages sqlite-db repo)))
-
-  ;; fetch all blocks, return block id and page id
-  (fetchAllBlocks
-   [_this repo]
-   (p/do!
-    (.ensure_init sqlite-db)
-    (.fetch_all_blocks sqlite-db repo)))
-
-  (fetchRecentJournals
-   [_this repo]
-   (p/do!
-    (.ensure_init sqlite-db)
-    (.fetch_recent_journals sqlite-db repo)))
-
-  (fetchInitData
-   [_this repo]
-   (p/do!
-    (.ensure_init sqlite-db)
-    (.fetch_init_data sqlite-db repo)))
-
-  (fetchBlocksExcluding
-   [_this repo excluding-uuids]
-   (p/do!
-    (.ensure_init sqlite-db)
-    (.fetch_blocks_excluding sqlite-db repo excluding-uuids))))
+            dt/write-transit-str)))))
 
 (defn init
   "web worker entry"

src/main/frontend/handler.cljs

@@ -49,7 +49,8 @@
             [frontend.db.listener :as db-listener]
             [cljs-bean.core :as bean]
             [frontend.handler.test :as test]
-            [frontend.db.rtc.op-mem-layer :as op-mem-layer]))
+            [frontend.db.rtc.op-mem-layer :as op-mem-layer]
+            [frontend.persist-db.browser :as db-browser]))
 
 (defn- set-global-error-notification!
   []
@@ -242,7 +243,8 @@
   (p/do!
    (when (mobile-util/native-platform?)
      (mobile/mobile-preinit))
-   (-> (p/let [repos (get-repos)
+   (-> (p/let [_ (db-browser/start-db-worker!)
+               repos (get-repos)
                _ (state/set-repos! repos)
                _ (mobile-util/hide-splash) ;; hide splash as early as ui is stable
                _ (restore-and-setup! repos)]

src/main/frontend/persist_db/browser.cljs

@@ -1,5 +1,5 @@
 (ns frontend.persist-db.browser
-  "Browser db persist support, using @logseq/sqlite.
+  "Browser db persist support, using @logseq/sqlite-wasm.
 
    This interface uses clj data format as input."
   (:require ["comlink" :as Comlink]
@@ -8,113 +8,64 @@
             [frontend.config :as config]
             [promesa.core :as p]
             [frontend.util :as util]
-            [frontend.handler.notification :as notification]
-            [clojure.string :as string]))
+            [frontend.handler.notification :as notification]))
 
 (defonce *sqlite (atom nil))
-(defonce *inited (atom false))
 
-(when-not (or config/publishing? util/node-test?)
-  (defonce _do_not_reload_worker
-    (let [worker-url (if (util/electron?)
-                       "js/db-worker.js"
-                       "/static/js/db-worker.js")
-          sqlite-url (if (util/electron?)
-                       "logseq_sqlite_bg.wasm"
-                       "/static/js/logseq_sqlite_bg.wasm")
-          worker (try
-                   (js/Worker. (str worker-url "?url=" sqlite-url))
-                   (catch js/Error e
-                     (js/console.error "worker error", e)
-                     nil))
-          ^js sqlite (Comlink/wrap worker)]
-      (p/catch (.init sqlite)
-               (fn [error]
-                 (notification/show! [:div (str "init error: " error)] :error))) ;; load wasm
-      (reset! *sqlite sqlite))))
-
-
-(defn- ensure-sqlite-init
+(defn start-db-worker!
   []
-  (if @*inited
-    (p/resolved @*sqlite)
-    (js/Promise. (fn [resolve reject]
-                   (let [timer (atom nil)
-                         timer' (js/setInterval (fn []
-                                                  (p/let [inited (.inited ^js @*sqlite)]
-                                                    (when inited
-                                                      (p/let [version (.getVersion ^js @*sqlite)
-                                                              support (.supportOPFS ^js @*sqlite)]
-                                                        (prn :init-sqlite version :opfs-support support))
-                                                      (js/clearInterval @timer)
-                                                      (reset! *inited true)
-                                                      (resolve @*sqlite))))
-                                                200)
-                         _ (reset! timer timer')]
-                     (js/setTimeout (fn []
-                                      (js/clearInterval timer)
-                                      (reject nil)) ;; cannot init
-                                    20000))))))
-
-(comment
-  (defn dev-stop!
-    "For dev env only, stop opfs backend, close all sqlite connections and OPFS sync access handles."
-    []
-    (println "[persis-db] Dev: close all sqlite connections")
-    (when-not (util/electron?)
-      (when @*sqlite
-        (.unsafeDevCloseAll ^js @*sqlite)))))
-
+  (when-not (or config/publishing? util/node-test?)
+    (p/let [worker-url (if (util/electron?)
+                         "js/db-worker.js"
+                         "/static/js/db-worker.js")
+            worker (js/Worker. worker-url)
+            _ (prn :debug :worker worker)
+            ^js sqlite (Comlink/wrap worker)
+            ;; _ (.init sqlite)
+            ]
+      (prn :debug :sqlite)
+      (js/console.dir sqlite)
+      (reset! *sqlite sqlite))))
 
 (defrecord InBrowser []
   protocol/PersistentDB
   (<new [_this repo]
-    (prn ::new-repo repo)
-    (-> (p/let [^js sqlite (ensure-sqlite-init)]
+    (prn :debug ::new-repo repo)
+    (-> (p/let [^js sqlite @*sqlite]
           (.newDB sqlite repo))
         (p/catch (fn [error]
-                   (if (string/includes? (str error) "NoModificationAllowedError")
-                     (notification/show! [:div (str "Avoid opening the same graph in multi-tabs. Error: " error)] :error)
-                     (notification/show! [:div (str "SQLiteDB creation error: " error)] :error))
-
+                   (js/console.error error)
+                   (notification/show! [:div (str "SQLiteDB creation error: " error)] :error)
                    nil))))
 
   (<list-db [_this]
-    (prn :debug :ensure-sqlite-init (js/Date.))
-    (-> (p/let [^js sqlite (ensure-sqlite-init)]
-          (prn :debug :list-db (js/Date.))
-          (.listDB sqlite))
-        (p/catch (fn [error]
-                   (prn :debug :list-db-error (js/Date.))
-                   (notification/show! [:div (str "SQLiteDB error: " error)] :error)
-                   []))))
+    (when-let [^js sqlite @*sqlite]
+      (-> (.listDB sqlite)
+          (p/catch (fn [error]
+                     (prn :debug :list-db-error (js/Date.))
+                     (notification/show! [:div (str "SQLiteDB error: " error)] :error)
+                     [])))))
 
   (<unsafe-delete [_this repo]
-    (p/let [^js sqlite (ensure-sqlite-init)]
-      (.unsafeUnlinkDB sqlite repo)))
+    (p/let [^js sqlite @*sqlite]
+      ;; (.unsafeUnlinkDB sqlite repo)
+      ))
 
   (<transact-data [_this repo tx-data tx-meta]
-    (prn ::transact-data repo (count tx-data) (count tx-meta))
+    (prn :debug ::transact-data repo (count tx-data) (count tx-meta))
     (p->c
-     (p/let [^js sqlite (ensure-sqlite-init)
+     (p/let [^js sqlite @*sqlite
              _ (.transact sqlite repo (pr-str tx-data) (pr-str tx-meta))]
        nil)))
 
   (<fetch-initital-data [_this repo _opts]
-    (prn ::fetch-initital-data repo)
-    (-> (p/let [^js sqlite (ensure-sqlite-init)
+    (prn ::fetch-initital-data repo @*sqlite)
+    (-> (p/let [^js sqlite @*sqlite
                 ;; <fetch-initital-data is called when init/re-loading graph
                 ;; the underlying DB should be opened
                 _ (.newDB sqlite repo)]
           (.getInitialData sqlite repo))
         (p/catch (fn [error]
-                   (prn ::fuck-error)
-                   (if (string/includes? (str error) "NoModificationAllowedError")
-                     (notification/show! [:div (str "Avoid opening the same graph in multi-tabs. Error: " error)] :error)
-                     (notification/show! [:div (str "SQLiteDB fetch error: " error)] :error))
-
-                   {}))))
-
-  (<fetch-blocks-excluding [_this repo exclude-uuids _opts]
-    (p/let [^js sqlite (ensure-sqlite-init)]
-      (.fetchBlocksExcluding sqlite repo (clj->js exclude-uuids)))))
+                   (prn :debug :fetch-initial-data-error)
+                   (js/console.error error)
+                   (notification/show! [:div (str "SQLiteDB fetch error: " error)] :error) {})))))

src/main/frontend/persist_db/protocol.cljs

@@ -8,5 +8,4 @@
   (<new [this repo])
   (<unsafe-delete [this repo])
   (<transact-data [this repo tx-data tx-meta] "Transact data to db")
-  (<fetch-initital-data [this repo opts])
-  (<fetch-blocks-excluding [this repo exclude-uuids opts]))
+  (<fetch-initital-data [this repo opts]))