perf: idb batch writes for importing files from local directories

src/main/frontend/handler/web/nfs.cljs

@@ -2,6 +2,7 @@
   "The File System Access API, https://web.dev/file-system-access/."
   (:require [cljs-bean.core :as bean]
             [promesa.core :as p]
+            [medley.core :as medley]
             [goog.object :as gobj]
             [goog.dom :as gdom]
             [frontend.util :as util]
@@ -57,12 +58,30 @@
                        (keyword (util/get-file-ext (:file/path file)))))
           files))
 
+(defn- set-batch!
+  [handles]
+  (let [handles (map (fn [[path handle]]
+                       {:key path
+                        :value handle}) handles)]
+    (idb/set-batch! handles)))
+
+(defn- set-files-aux!
+  [handles]
+  (if (seq handles)
+    (let [[h t] (split-at 50 handles)]
+      (p/let [_ (p/promise (fn [_]
+                             (js/setTimeout (fn []
+                                              (p/resolved nil)) 10)))
+              _ (set-batch! h)]
+        (when (seq t)
+          (set-files-aux! t))))))
+
 (defn- set-files!
   [handles]
   (doseq [[path handle] handles]
     (let [handle-path (str config/local-handle-prefix path)]
-      (idb/set-item! handle-path handle)
-      (fs/add-nfs-file-handle! handle-path handle))))
+      (fs/add-nfs-file-handle! handle-path handle)))
+  (set-files-aux! handles))
 
 (defn ls-dir-files
   []
@@ -223,7 +242,7 @@
                                         (rename-f "modify" modified))]
                              (when (or (and (seq diffs) (seq modified-files))
                                        (seq diffs) ; delete
-                                       )
+)
                                (repo-handler/load-repo-to-db! repo
                                                               {:diffs diffs
                                                                :nfs-files modified-files})))))))))

src/main/frontend/idb.cljs

@@ -1,26 +1,24 @@
 (ns frontend.idb
-  (:require ["localforage" :as localforage]
-            [cljs-bean.core :as bean]
+  (:require [cljs-bean.core :as bean]
             [goog.object :as gobj]
             [promesa.core :as p]
             [clojure.string :as string]
             [frontend.config :as config]
             [frontend.util :as util]
-            [frontend.storage :as storage]))
+            [frontend.storage :as storage]
+            ["/frontend/idbkv" :as idb-keyval :refer [Store]]))
+
 
 ;; offline db
-(def store-name "dbs")
-(.config localforage
-         (bean/->js
-          {:name "logseq-datascript"
-           :version 1.0
-           :storeName store-name}))
 
-(defonce localforage-instance (.createInstance localforage store-name))
+;; To maintain backward compatibility
+
+
+(defonce store (Store. "localforage" "keyvaluepairs" 2))
 
 (defn clear-idb!
   []
-  (p/let [_ (.clear localforage-instance)
+  (p/let [_ (idb-keyval/clear store)
           dbs (js/window.indexedDB.databases)]
     (doseq [db dbs]
       (js/window.indexedDB.deleteDatabase (gobj/get db "name")))))
@@ -33,21 +31,26 @@
 (defn remove-item!
   [key]
   (when key
-    (.removeItem localforage-instance key)))
+    (idb-keyval/del key store)))
 
 (defn set-item!
   [key value]
   (when key
-    (.setItem localforage-instance key value)))
+    (idb-keyval/set key value store)))
+
+(defn set-batch!
+  [items]
+  (when (seq items)
+    (idb-keyval/setBatch (clj->js items) store)))
 
 (defn get-item
   [key]
   (when key
-    (.getItem localforage-instance key)))
+    (idb-keyval/get key store)))
 
 (defn get-keys
   []
-  (.keys localforage-instance))
+  (idb-keyval/keys store))
 
 (defn get-nfs-dbs
   []

src/main/frontend/idbkv.js

@@ -0,0 +1,142 @@
+'use strict';
+
+Object.defineProperty(exports, '__esModule', { value: true });
+
+class Store {
+    constructor(dbName = 'keyval-store', storeName = 'keyval', version = 1) {
+        this.storeName = storeName;
+        this._dbName = dbName;
+        this._storeName = storeName;
+        this._version = version;
+        this.id = `dbName:${dbName};;storeName:${storeName}`;
+        this._init();
+    }
+    _init() {
+        if (this._dbp) {
+            return;
+        }
+        this._dbp = new Promise((resolve, reject) => {
+            const openreq = indexedDB.open(this._dbName, this._version);
+            openreq.onerror = () => reject(openreq.error);
+            openreq.onsuccess = () => resolve(openreq.result);
+            // First time setup: create an empty object store
+            openreq.onupgradeneeded = () => {
+                openreq.result.createObjectStore(this._storeName);
+            };
+        });
+    }
+    _withIDBStore(type, callback) {
+        this._init();
+        return this._dbp.then(db => new Promise((resolve, reject) => {
+            const transaction = db.transaction(this.storeName, type);
+            transaction.oncomplete = () => resolve();
+            transaction.onabort = transaction.onerror = () => reject(transaction.error);
+            callback(transaction.objectStore(this.storeName));
+        }));
+    }
+    _close() {
+        this._init();
+        return this._dbp.then(db => {
+            db.close();
+            this._dbp = undefined;
+        });
+    }
+}
+class Batcher {
+    constructor(executor) {
+        this.executor = executor;
+        this.items = [];
+    }
+  async process() {
+        const toProcess = this.items;
+        this.items = [];
+        await this.executor(toProcess.map(({ item }) => item));
+        toProcess.map(({ onProcessed }) => onProcessed());
+        if (this.items.length) {
+            this.ongoing = this.process();
+        }
+        else {
+            this.ongoing = undefined;
+        }
+    }
+    async queue(item) {
+        const result = new Promise((resolve) => this.items.push({ item, onProcessed: resolve }));
+        if (!this.ongoing)
+            this.ongoing = this.process();
+        return result;
+    }
+}
+let store;
+function getDefaultStore() {
+    if (!store)
+        store = new Store();
+    return store;
+}
+function get(key, store = getDefaultStore()) {
+    let req;
+    return store._withIDBStore('readwrite', store => {
+        req = store.get(key);
+    }).then(() => req.result);
+}
+const setBatchers = {};
+function set(key, value, store = getDefaultStore()) {
+    if (!setBatchers[store.id]) {
+        setBatchers[store.id] = new Batcher((items) => store._withIDBStore('readwrite', store => {
+            for (const item of items) {
+                store.put(item.value, item.key);
+            }
+        }));
+    }
+    return setBatchers[store.id].queue({ key, value });
+}
+function setBatch(items, store = getDefaultStore()) {
+  return store._withIDBStore('readwrite', store => {
+    for (const item of items) {
+            store.put(item.value, item.key);
+    }
+  });
+}
+function update(key, updater, store = getDefaultStore()) {
+    return store._withIDBStore('readwrite', store => {
+        const req = store.get(key);
+        req.onsuccess = () => {
+            store.put(updater(req.result), key);
+        };
+    });
+}
+function del(key, store = getDefaultStore()) {
+    return store._withIDBStore('readwrite', store => {
+        store.delete(key);
+    });
+}
+function clear(store = getDefaultStore()) {
+    return store._withIDBStore('readwrite', store => {
+        store.clear();
+    });
+}
+function keys(store = getDefaultStore()) {
+    const keys = [];
+    return store._withIDBStore('readwrite', store => {
+        // This would be store.getAllKeys(), but it isn't supported by Edge or Safari.
+        // And openKeyCursor isn't supported by Safari.
+        (store.openKeyCursor || store.openCursor).call(store).onsuccess = function () {
+            if (!this.result)
+                return;
+            keys.push(this.result.key);
+            this.result.continue();
+        };
+    }).then(() => keys);
+}
+function close(store = getDefaultStore()) {
+    return store._close();
+}
+
+exports.Store = Store;
+exports.get = get;
+exports.set = set;
+exports.setBatch = setBatch;
+exports.update = update;
+exports.del = del;
+exports.clear = clear;
+exports.keys = keys;
+exports.close = close;