Merge branch 'enhance/write-to-disk' into refactor/worker-search

.clj-kondo/config.edn

@@ -104,6 +104,7 @@
              frontend.modules.outliner.core outliner-core
              frontend.mobile.util mobile-util
              frontend.page page
+             frontend.persist-db persist-db
              frontend.schema.handler.common-config common-config-schema
              frontend.search search
              frontend.state state

deps.edn

@@ -3,8 +3,8 @@
  {org.clojure/clojure                   {:mvn/version "1.11.1"}
   rum/rum                               {:mvn/version "0.12.9"}
 
-  datascript/datascript        {:git/url "https://github.com/logseq/datascript" ;; fork
-                                :sha     "ec937bc2bc932f0e1fdbbc1f67413d55c18ddc4a"}
+  datascript/datascript                 {:git/url "https://github.com/logseq/datascript" ;; fork
+                                         :sha     "15234c650df20844ec089c1738d28c00eccacd6e"}
 
   datascript-transit/datascript-transit {:mvn/version "0.3.0"}
   borkdude/rewrite-edn                  {:mvn/version "0.4.7"}

deps/common/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2"
   },
   "scripts": {
     "test": "yarn nbb-logseq -cp test -m nextjournal.test-runner"

deps/common/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1":
-  version "1.2.173-feat-db-v1"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2":
+  version "1.2.173-feat-db-v2"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/db/.carve/config.edn

@@ -1,5 +1,6 @@
 {:paths ["src"]
  :api-namespaces [logseq.db.sqlite.db
+                  logseq.db.sqlite.common-db
                   logseq.db.sqlite.rtc
                   logseq.db.sqlite.util
                   logseq.db.sqlite.cli

deps/db/deps.edn

@@ -1,7 +1,7 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "3b578bb4b39efda730cdae5f8e679060101b5110"}
+                         :sha     "15234c650df20844ec089c1738d28c00eccacd6e"}
   com.cognitect/transit-cljs   {:mvn/version "0.8.280"}
   cljs-bean/cljs-bean          {:mvn/version "1.5.0"}}
  :aliases

deps/db/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2"
   },
   "dependencies": {
     "better-sqlite3": "8.0.1"

deps/db/script/query.cljs

@@ -5,7 +5,6 @@
     (:require [datascript.core :as d]
               [clojure.edn :as edn]
               [logseq.db.sqlite.db :as sqlite-db]
-              [logseq.db.sqlite.cli :as sqlite-cli]
               [logseq.db.frontend.rules :as rules]
               [nbb.core :as nbb]
               ["path" :as path]
@@ -15,18 +14,17 @@
   "The db graph bare version of gp-cli/parse-graph"
   [graph-name]
   (let [graphs-dir (path/join (os/homedir) "logseq/graphs")]
-    (sqlite-db/open-db! graphs-dir graph-name)
-    (sqlite-cli/read-graph graph-name)))
+    (sqlite-db/open-db! graphs-dir graph-name)))
 
 (defn -main [args]
-  (when (not= 2 (count args))
+  (when (< (count args) 2)
     (println "Usage: $0 GRAPH QUERY")
     (js/process.exit 1))
   (let [[graph-name query*] args
         conn (read-graph graph-name)
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries
         results (mapv first (d/q query @conn (rules/extract-rules rules/db-query-dsl-rules)))]
-    #_(println "DB contains" (count (d/datoms @conn :eavt)) "datoms")
+    (when ((set args) "-v") (println "DB contains" (count (d/datoms @conn :eavt)) "datoms"))
     (prn results)))
 
 (when (= nbb/*file* (:file (meta #'-main)))

deps/db/script/validate_client_db.cljs

@@ -1,8 +1,7 @@
 (ns validate-client-db
   "Script that validates the datascript db of a DB graph
    NOTE: This script is also used in CI to confirm our db's schema is up to date"
-  (:require [logseq.db.sqlite.cli :as sqlite-cli]
-            [logseq.db.sqlite.db :as sqlite-db]
+  (:require [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [datascript.core :as d]
             [clojure.string :as string]
@@ -93,11 +92,10 @@
                               (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
                           ((juxt node-path/dirname node-path/basename) graph-dir'))
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        _ (try (sqlite-db/open-db! dir db-name)
-               (catch :default e
-                 (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
-                 (js/process.exit 1)))
-        conn (sqlite-cli/read-graph db-name)
+        conn (try (sqlite-db/open-db! dir db-name)
+                  (catch :default e
+                    (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
+                    (js/process.exit 1)))
         datoms (d/datoms @conn :eavt)
         ent-maps (db-malli-schema/datoms->entity-maps datoms)]
     (println "Read graph" (str db-name " with " (count datoms) " datoms, "

deps/db/src/logseq/db/sqlite/cli.cljs

@@ -1,20 +1,9 @@
 (ns ^:node-only logseq.db.sqlite.cli
   "Primary ns to interact with DB graphs with node.js based CLIs"
-  (:require [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
-            ["fs" :as fs]
+  (:require ["fs" :as fs]
             ["path" :as node-path]))
 
 (defn db-graph-directory?
   "Returns boolean indicating if the given directory is a DB graph"
   [graph-dir]
-  (fs/existsSync (node-path/join graph-dir "db.sqlite")))
-
-(defn read-graph
-  "Reads a given sqlite db and returns a datascript connection of its contents.
-   The sqlite db is assumed to have already been opened by sqlite-db/open-db!"
-  [db-name]
-  (-> (sqlite-db/get-conn db-name)
-      deref
-      sqlite-common-db/get-initial-data
-      sqlite-common-db/restore-initial-data))
+  (fs/existsSync (node-path/join graph-dir "db.sqlite")))

deps/db/src/logseq/db/sqlite/db.cljs

@@ -4,6 +4,7 @@
             ["better-sqlite3" :as sqlite3]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
             ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
+            #_:clj-kondo/ignore
             [datascript.core :as d]
             [datascript.storage :refer [IStorage]]
             [goog.object :as gobj]
@@ -11,14 +12,14 @@
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.util :as sqlite-util]))
 
+;; Reference same sqlite default class in cljs + nbb without needing .cljc
+(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
+
 ;; sqlite databases
 (defonce databases (atom nil))
 ;; datascript conns
 (defonce conns (atom nil))
 
-;; Reference same sqlite default class in cljs + nbb without needing .cljc
-(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
-
 (defn close!
   []
   (when @databases
@@ -34,12 +35,6 @@
   [repo]
   (get @conns (sanitize-db-name repo)))
 
-(defn get-db-full-path
-  [graphs-dir db-name]
-  (let [db-name' (sanitize-db-name db-name)
-        graph-dir (node-path/join graphs-dir db-name')]
-    [db-name' (node-path/join graph-dir "db.sqlite")]))
-
 (defn query
   [db sql]
   (let [stmt (.prepare db sql)]
@@ -47,12 +42,16 @@
 
 (defn upsert-addr-content!
   "Upsert addr+data-seq"
-  [db data]
+  [db data delete-addrs]
   (let [insert (.prepare db "INSERT INTO kvs (addr, content) values (@addr, @content) on conflict(addr) do update set content = @content")
+        delete (.prepare db "DELETE from kvs where addr = ?")
         insert-many (.transaction ^object db
                                   (fn [data]
                                     (doseq [item data]
-                                      (.run ^object insert item))))]
+                                      (.run ^object insert item))
+                                    (doseq [addr delete-addrs]
+                                      (when addr
+                                        (.run ^object delete addr)))))]
     (insert-many data)))
 
 (defn restore-data-from-addr
@@ -65,7 +64,7 @@
   "Creates a datascript storage for sqlite. Should be functionally equivalent to db-worker/new-sqlite-storage"
   [db]
   (reify IStorage
-    (-store [_ addr+data-seq]
+    (-store [_ addr+data-seq delete-addrs]
       (let [data (->>
                   (map
                    (fn [[addr data]]
@@ -73,12 +72,15 @@
                           :content (pr-str data)})
                    addr+data-seq)
                   (to-array))]
-        (upsert-addr-content! db data)))
+        (upsert-addr-content! db data delete-addrs)))
     (-restore [_ addr]
       (let [content (restore-data-from-addr db addr)]
         (edn/read-string content)))))
 
 (defn open-db!
+  "For a given database name, opens a sqlite db connection for it, creates
+  needed sqlite tables if not created and returns a datascript connection that's
+  connected to the sqlite db"
   [graphs-dir db-name]
   (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
         db (new sqlite db-full-path nil)
@@ -89,15 +91,10 @@
     (swap! databases assoc db-sanitized-name db)
     (let [storage (new-sqlite-storage db)
           conn (sqlite-common-db/get-storage-conn storage schema)]
-      (swap! conns assoc db-sanitized-name conn)))
-  nil)
+      (swap! conns assoc db-sanitized-name conn)
+      conn)))
 
-;; TODO: Remove as it looks unused
 (defn transact!
   [repo tx-data tx-meta]
   (when-let [conn (get-conn repo)]
-    (try
-      (d/transact! conn tx-data tx-meta)
-      (catch :default e
-        (prn :debug :error)
-        (js/console.error e)))))
+    (d/transact! conn tx-data tx-meta)))

deps/db/test/logseq/db/sqlite/db_test.cljs

@@ -24,11 +24,11 @@
 (deftest get-initial-data
   (testing "Fetches a defined block"
     (create-graph-dir "tmp/graphs" "test-db")
-    (sqlite-db/open-db! "tmp/graphs" "test-db")
-    (let [blocks [{:block/uuid (random-uuid)
+    
+    (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
+          blocks [{:block/uuid (random-uuid)
                    :file/path "logseq/config.edn"
                    :file/content "{:foo :bar}"}]
-          conn* (sqlite-db/get-conn "test-db")
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
           conn (-> (sqlite-common-db/get-initial-data @conn*)
@@ -42,8 +42,8 @@
 (deftest restore-initial-data
   (testing "Restore a journal page with its block"
     (create-graph-dir "tmp/graphs" "test-db")
-    (sqlite-db/open-db! "tmp/graphs" "test-db")
-    (let [page-uuid (random-uuid)
+    (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
+          page-uuid (random-uuid)
           block-uuid (random-uuid)
           created-at (js/Date.now)
           blocks [{:db/id 100001
@@ -58,7 +58,6 @@
                    :block/page {:db/id 100001}
                    :block/created-at created-at
                    :block/updated-at created-at}]
-          conn* (sqlite-db/get-conn "test-db")
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
           conn (-> (sqlite-common-db/get-initial-data @conn*)

deps/db/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1":
-  version "1.2.173-feat-db-v1"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2":
+  version "1.2.173-feat-db-v2"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/graph-parser/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2",
     "better-sqlite3": "8.0.1"
   },
   "dependencies": {

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -5,7 +5,6 @@
             [clojure.string :as string]
             [datascript.core :as d]
             [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db :as ldb]
             [clojure.set :as set]
             ["fs" :as fs]
@@ -127,10 +126,9 @@
   "Creates a sqlite-based db graph given a map of pages to blocks and returns a datascript db.
    Blocks in a map can only be top-level blocks with no referenced content"
   [dir db-name pages-to-blocks]
-  (sqlite-db/open-db! dir db-name)
-  (let [frontend-blocks (create-frontend-blocks pages-to-blocks)
-        _ (sqlite-db/transact! db-name frontend-blocks {})
-        conn (sqlite-cli/read-graph db-name)]
+  (let [conn (sqlite-db/open-db! dir db-name)
+        frontend-blocks (create-frontend-blocks pages-to-blocks)
+        _ (d/transact! conn frontend-blocks)]
     (ldb/create-default-pages! conn {:db-graph? true})
     @conn))
 

deps/graph-parser/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1":
-  version "1.2.173-feat-db-v1"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2":
+  version "1.2.173-feat-db-v2"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/outliner/.carve/ignore

@@ -1,2 +1,2 @@
 ;; API fn
-logseq.outliner.cli.persist-graph/add-listener
+logseq.outliner.cli.pipeline/add-listener

deps/outliner/deps.edn

@@ -1,7 +1,7 @@
 {:deps
  ;; External deps should be kept in sync with https://github.com/logseq/nbb-logseq/blob/main/bb.edn
  {datascript/datascript {:git/url "https://github.com/logseq/datascript" ;; fork
-                         :sha     "3b578bb4b39efda730cdae5f8e679060101b5110"}
+                         :sha     "15234c650df20844ec089c1738d28c00eccacd6e"}
   logseq/db             {:local/root "../db"}
   com.cognitect/transit-cljs {:mvn/version "0.8.280"}}
  :aliases

deps/outliner/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2"
   },
   "dependencies": {
     "better-sqlite3": "8.0.1"

deps/outliner/script/transact.cljs

@@ -1,7 +1,6 @@
 (ns transact
   "This script generically runs transactions against the queried blocks"
-  (:require [logseq.outliner.cli.persist-graph :as persist-graph]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+  (:require [logseq.outliner.cli.pipeline :as cli-pipeline]
             [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.frontend.rules :as rules]
             [datascript.core :as d]
@@ -20,8 +19,7 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        _ (sqlite-db/open-db! dir db-name)
-        conn (sqlite-cli/read-graph db-name)
+        conn (sqlite-db/open-db! dir db-name)
         ;; find blocks to update
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries
         transact-fn (edn/read-string transact-fn*)
@@ -35,7 +33,7 @@
           (println "With the following blocks updated:")
           (prn (map #(select-keys (d/entity @conn %) [:block/name :block/content]) blocks-to-update)))
       (do
-        (persist-graph/add-listener conn db-name)
+        (cli-pipeline/add-listener conn)
         (d/transact! conn update-tx)
         (println "Updated" (count update-tx) "block(s) for graph" (str db-name "!"))))))
 

deps/outliner/src/logseq/outliner/cli/persist_graph.cljs

@@ -1,32 +0,0 @@
-(ns ^:node-only logseq.outliner.cli.persist-graph
-  "This ns allows DB graphs to persist datascript changes to their respective
-  sqlite db. Since changes are persisted, this can be used to create or update graphs.
-   Known limitations:
-   * Deleted blocks don't update effected :block/tx-id"
-  (:require [datascript.core :as d]
-            [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.outliner.datascript-report :as ds-report]
-            [logseq.outliner.pipeline :as outliner-pipeline]))
-
-(defn- invoke-hooks
-  "Modified copy of frontend.modules.outliner.pipeline/invoke-hooks that doesn't
-  handle :block/tx-id"
-  [conn tx-report]
-  (when (not (get-in tx-report [:tx-meta :replace?]))
-    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
-          block-path-refs-tx (outliner-pipeline/compute-block-path-refs-tx tx-report blocks)]
-      (d/transact! conn block-path-refs-tx {:replace? true})
-      ;; frontend also passes original tx-report
-      tx-report)))
-
-(defn- update-sqlite-db
-  "Same as :db-transact-data defmethod in electron.handler"
-  [db-name tx-report]
-  (sqlite-db/transact! db-name (:tx-data tx-report) (:tx-meta tx-report)))
-
-(defn add-listener
-  "Adds a listener to the datascript connection to persist changes to the given
-  sqlite db name"
-  [conn db-name]
-  (d/listen! conn :persist-to-sqlite (fn persist-to-sqlite [tx-report]
-                                       (update-sqlite-db db-name (invoke-hooks conn tx-report)))))

deps/outliner/src/logseq/outliner/cli/pipeline.cljs

@@ -0,0 +1,23 @@
+(ns ^:node-only logseq.outliner.cli.pipeline
+  "This ns provides a datascript listener for DB graphs to add additional changes
+   that the frontend also adds per transact.
+   Known limitations:
+   * Deleted blocks don't update effected :block/tx-id"
+  (:require [datascript.core :as d]
+            [logseq.outliner.datascript-report :as ds-report]
+            [logseq.outliner.pipeline :as outliner-pipeline]))
+
+(defn- invoke-hooks
+  "Modified copy of frontend.modules.outliner.pipeline/invoke-hooks that doesn't
+  handle :block/tx-id"
+  [conn tx-report]
+  (when (not (get-in tx-report [:tx-meta :replace?]))
+    (let [{:keys [blocks]} (ds-report/get-blocks-and-pages tx-report)
+          block-path-refs-tx (outliner-pipeline/compute-block-path-refs-tx tx-report blocks)]
+      (d/transact! conn block-path-refs-tx {:replace? true}))))
+
+(defn add-listener
+  "Adds a listener to the datascript connection to add additional changes from outliner.pipeline"
+  [conn]
+  (d/listen! conn :pipeline-updates (fn pipeline-updates [tx-report]
+                                      (invoke-hooks conn tx-report))))

deps/outliner/src/logseq/outliner/pipeline.cljs

@@ -1,8 +1,6 @@
 (ns logseq.outliner.pipeline
   "Core fns for use with frontend.modules.outliner.pipeline"
-  (:require [logseq.db.frontend.schema :as db-schema]
-            [datascript.core :as d]
-            [cognitect.transit :as t]
+  (:require [datascript.core :as d]
             [clojure.set :as set]))
 
 (defn filter-deleted-blocks
@@ -13,23 +11,6 @@
        (:v d)))
    datoms))
 
-(defn- datom->av-vector
-  [db datom]
-  (let [a (:a datom)
-        v (:v datom)
-        v' (cond
-             (contains? db-schema/ref-type-attributes a)
-             (when-some [block-uuid-datom (first (d/datoms db :eavt v :block/uuid))]
-               [:block/uuid (str (:v block-uuid-datom))])
-
-             (and (= :block/uuid a) (uuid? v))
-             (str v)
-
-             :else
-             v)]
-    (when (some? v')
-      [a v'])))
-
 ;; non recursive query
 (defn ^:api get-block-parents
   [db block-id {:keys [depth] :or {depth 100}}]

deps/outliner/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1":
-  version "1.2.173-feat-db-v1"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2":
+  version "1.2.173-feat-db-v2"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

deps/publishing/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1",
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2",
     "mldoc": "^1.5.1"
   },
   "dependencies": {

deps/publishing/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1":
-  version "1.2.173-feat-db-v1"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2":
+  version "1.2.173-feat-db-v2"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

package.json

@@ -101,7 +101,7 @@
         "@logseq/capacitor-file-sync": "5.0.1",
         "@logseq/diff-merge": "0.2.2",
         "@logseq/react-tweet-embed": "1.3.1-1",
-        "@logseq/sqlite-wasm": "=0.0.7",
+        "@logseq/sqlite-wasm": "=0.1.0",
         "@radix-ui/colors": "^0.1.8",
         "@sentry/react": "^6.18.2",
         "@sentry/tracing": "^6.18.2",

scripts/README.md

@@ -45,7 +45,5 @@ Created graph schema!
 
 #### Update graph scripts
 
-For database graphs, it is possible to update graphs with the
-`logseq.outliner.cli.persist-graph`.
-ns. This ns makes it easy to write scripts that update graphs using datascript
-and logseq's schema. TODO
+For database graphs, it is recommended to use
+`logseq.outliner.cli.pipeline/add-listener!` when updating graphs.  TODO

scripts/package.json

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "private": true,
   "devDependencies": {
-    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v1"
+    "@logseq/nbb-logseq": "logseq/nbb-logseq#feat-db-v2"
   },
   "dependencies": {
     "better-sqlite3": "8.0.1",

scripts/src/logseq/tasks/db_graph/create_graph.cljs

@@ -7,7 +7,7 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.frontend.property.util :as db-property-util]
-            [logseq.outliner.cli.persist-graph :as persist-graph]
+            [logseq.outliner.cli.pipeline :as cli-pipeline]
             [logseq.db :as ldb]
             [clojure.string :as string]
             [datascript.core :as d]
@@ -34,10 +34,9 @@
   transacts initial data"
   [dir db-name]
   (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
-  (sqlite-db/open-db! dir db-name)
   ;; Same order as frontend.db.conn/start!
-  (let [conn (ldb/start-conn :create-default-pages? false)]
-    (persist-graph/add-listener conn db-name)
+  (let [conn (sqlite-db/open-db! dir db-name)]
+    (cli-pipeline/add-listener conn)
     (ldb/create-default-pages! conn {:db-graph? true})
     (setup-init-data conn)
     conn))

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -15,7 +15,8 @@
         ["logseq.db.sqlite." "logseq.db.frontend.property" "logseq.db.frontend.malli-schema"
          "electron.db"
          "frontend.handler.db-based."
-         "frontend.components.property" "frontend.components.class" "frontend.components.db-based"]))
+         "frontend.components.property" "frontend.components.class" "frontend.components.db-based"
+         "frontend.persist-db"]))
 
 (def file-graph-ns
   "Namespaces or parent namespaces _only_ for file graphs"
@@ -34,6 +35,9 @@
    "src/main/frontend/components/property.cljs"
    "src/main/frontend/components/property"
    "src/main/frontend/components/db_based"
+   ;; TODO: Enable this when run-export-periodically is deleted or moved out of the ns
+   #_"src/main/frontend/persist_db.cljs"
+   "src/main/frontend/persist_db"
    "src/electron/electron/db.cljs"])
 
 (def file-graph-paths

scripts/src/logseq/tasks/dev/publishing.cljs

@@ -23,8 +23,7 @@
 
 (defn- publish-db-graph [static-dir graph-dir output-path]
   (let [db-name (node-path/basename graph-dir)
-        _ (sqlite-db/open-db! (node-path/dirname graph-dir) db-name)
-        conn (sqlite-cli/read-graph db-name)
+        conn (sqlite-db/open-db! (node-path/dirname graph-dir) db-name)
         repo-config (-> (d/q '[:find ?content
                                :where [?b :file/path "logseq/config.edn"] [?b :file/content ?content]]
                              @conn)

scripts/yarn.lock

@@ -2,9 +2,9 @@
 # yarn lockfile v1
 
 
-"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v1":
-  version "1.2.173-feat-db-v1"
-  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/e4910dfb12043404c97962d8faab3a946ab89f81"
+"@logseq/nbb-logseq@logseq/nbb-logseq#feat-db-v2":
+  version "1.2.173-feat-db-v2"
+  resolved "https://codeload.github.com/logseq/nbb-logseq/tar.gz/9e891fd8a8a33ec78dd96a72bbdded191e1f0c1b"
   dependencies:
     import-meta-resolve "^2.1.0"
 

src/electron/electron/db.cljs

@@ -5,6 +5,8 @@
             ["electron" :refer [app]]
             ;; [electron.logger :as logger]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
+            [electron.logger :as logger]
+            [logseq.db.sqlite.db :as sqlite-db]
             [electron.backup-file :as backup-file]))
 
 (defn get-graphs-dir
@@ -23,6 +25,14 @@
     (fs/ensureDirSync graph-dir)
     graph-dir))
 
+(defn open-db!
+  [db-name]
+  (let [graphs-dir (get-graphs-dir)]
+    (try (sqlite-db/open-db! graphs-dir db-name)
+         (catch :default e
+           (js/console.error e)
+           (logger/error (str e ": " db-name))))))
+
 (defn save-db!
   [db-name data]
   (let [graph-dir (ensure-graph-dir! db-name)

src/electron/electron/handler.cljs

@@ -31,8 +31,10 @@
             [electron.window :as win]
             [electron.handler-interface :refer [handle]]
             [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db.sqlite.db :as sqlite-db]
             [logseq.common.graph :as common-graph]
-            [promesa.core :as p]))
+            [promesa.core :as p]
+            [clojure.edn :as edn]))
 
 (defmethod handle :mkdir [_window [_ dir]]
   (fs/mkdirSync dir))
@@ -304,6 +306,18 @@
   (db/ensure-graph-dir! repo)
   (db/save-db! repo data))
 
+(defmethod handle :db-open [_window [_ repo]]
+  (db/ensure-graph-dir! repo)
+  (db/open-db! repo)
+  nil)
+
+(defmethod handle :db-transact [_window [_ repo tx-data-str tx-meta-str]]
+  (when-let [conn (sqlite-db/get-conn repo)]
+    (let [tx-data (edn/read-string tx-data-str)
+          tx-meta (edn/read-string tx-meta-str)]
+      (sqlite-db/transact! repo tx-data tx-meta)
+      (:max-tx @conn))))
+
 ;; DB related IPCs End
 
 (defn clear-cache!
@@ -654,21 +668,6 @@
 (defmethod handle :system/info [^js _win _]
   {:home-dir (.homedir os)})
 
-(comment
-  ;; Needs to be called first for a new graph
-  (defmethod handle :db-new [_window [_ repo]]
-    (db/new-db! repo))
-
-  ;; Needs to be called first for an existing graph
-  (defmethod handle :get-initial-data [_window [_ repo _opts]]
-    (db/open-db! repo)
-    (dt/write-transit-str (sqlite-db/get-initial-data repo)))
-
-  (defmethod handle :db-transact-data [_window [_ repo data-str]]
-    (let [{:keys [tx-data tx-meta]} (reader/read-string data-str)]
-      (sqlite-db/transact! repo tx-data tx-meta)
-      nil)))
-
 (defn set-ipc-handler! [window]
   (let [main-channel "main"]
     (.handle ipcMain main-channel

src/electron/electron/state.cljs

@@ -16,9 +16,6 @@
          ;; window -> current graph
          :window/graph {}
 
-         ;; job to do when persistGraph is done on renderer
-         :window/once-persist-done nil
-
          ;; job to do when graph is loaded on renderer
          :window/once-graph-ready nil}))
 

src/main/electron/listener.cljs

@@ -20,7 +20,7 @@
             [logseq.graph-parser.util :as gp-util]
             [promesa.core :as p]
             [frontend.handler.property.util :as pu]
-            [frontend.persist-db :as persistent-db]))
+            [frontend.persist-db :as persist-db]))
 
 (defn- safe-api-call
   "Force the callback result to be nil, otherwise, ipc calls could lead to
@@ -32,7 +32,7 @@
   []
   (when-let [repo (state/get-current-repo)]
     (->
-     (p/let [_ (persistent-db/<export-db repo {})]
+     (p/let [_ (persist-db/<export-db repo {})]
        (ipc/ipc "persistent-dbs-saved"))
      (p/catch (fn [error]
                 (prn :debug :persist-db-failed :repo repo)
@@ -40,6 +40,7 @@
                 (notification/show! error :error))))))
 
 
+
 (defn listen-persistent-dbs!
   []
   (safe-api-call "persistent-dbs" (fn [_data] (persist-dbs!))))

src/main/frontend/components/export.cljs

@@ -1,6 +1,7 @@
 (ns frontend.components.export
   (:require [cljs-time.core :as t]
             ["/frontend/utils" :as utils]
+            [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
             [frontend.handler.export.text :as export-text]
@@ -26,9 +27,10 @@
       [:li.mb-4
        [:a.font-medium {:on-click #(export/export-repo-as-json-v2! current-repo)}
         (t :export-json)]]
-      [:li.mb-4
-       [:a.font-medium {:on-click #(export/export-repo-as-sqlite-db! current-repo)}
-        (t :export-sqlite-db)]]
+      (when (config/db-based-graph? current-repo)
+       [:li.mb-4
+        [:a.font-medium {:on-click #(export/export-repo-as-sqlite-db! current-repo)}
+         (t :export-sqlite-db)]])
       (when (util/electron?)
         [:li.mb-4
          [:a.font-medium {:on-click #(export/download-repo-as-html! current-repo)}

src/main/frontend/components/repo.cljs

@@ -150,7 +150,7 @@
     (p/let [multiple-windows? (ipc/ipc "graphHasMultipleWindows" (state/get-current-repo))]
       (reset! (::electron-multiple-windows? state) multiple-windows?))))
 
-(defn- repos-dropdown-links [repos current-repo *multiple-windows?]
+(defn- repos-dropdown-links [repos current-repo *multiple-windows? & {:as opts}]
   (let [switch-repos (if-not (nil? current-repo)
                        (remove (fn [repo] (= current-repo (:url repo))) repos) repos) ; exclude current repo
         repo-links (mapv
@@ -171,6 +171,8 @@
                                                          (ui/icon "cloud" {:size 18})])]
                            :hover-detail repo-url ;; show full path on hover
                            :options      {:on-click (fn [e]
+                                                      (when-let [on-click (:on-click opts)]
+                                                        (on-click e))
                                                       (if (gobj/get e "shiftKey")
                                                         (state/pub-event! [:graph/open-new-window url])
                                                         (if (or local? db-only?)
@@ -208,7 +210,7 @@
 
 (rum/defcs repos-dropdown < rum/reactive
   (rum/local false ::electron-multiple-windows?)
-  [state]
+  [state & {:as opts}]
   (let [multiple-windows? (::electron-multiple-windows? state)
         current-repo (state/sub :git/current-repo)
         login? (boolean (state/sub :auth/id-token))
@@ -218,7 +220,7 @@
             remotes (state/sub [:file-sync/remote-graphs :graphs])
             repos (if (and (seq remotes) login?)
                     (repo-handler/combine-local-&-remote-graphs repos remotes) repos)
-            links (repos-dropdown-links repos current-repo multiple-windows?)
+            links (repos-dropdown-links repos current-repo multiple-windows? opts)
             render-content (fn [{:keys [toggle-fn]}]
                              (let [remote? (:remote? (first (filter #(= current-repo (:url %)) repos)))
                                    repo-name (db/get-repo-name current-repo)
@@ -227,7 +229,7 @@
                                                      "Select a Graph")]
                                [:a.item.group.flex.items-center.p-2.text-sm.font-medium.rounded-md
 
-                                {:on-click (fn []
+                                {:on-click (fn [_e]
                                              (check-multiple-windows? state)
                                              (toggle-fn))
                                  :title    repo-name}       ;; show full path on hover

src/main/frontend/db/rtc/full_upload_download_graph.cljs

@@ -12,7 +12,8 @@
             [frontend.db.rtc.op-mem-layer :as op-mem-layer]
             [frontend.db.rtc.ws :refer [<send!]]
             [frontend.persist-db :as persist-db]
-            [logseq.db.frontend.schema :as db-schema]))
+            [logseq.db.frontend.schema :as db-schema]
+            [frontend.state :as state]))
 
 (def transit-r (transit/reader :json))
 
@@ -122,9 +123,10 @@
          blocks-with-page-id (fill-block-fields blocks*)]
      (<? (p->c (persist-db/<new repo)))
      (<? (p->c (persist-db/<transact-data repo blocks-with-page-id nil)))
+     (<? (p->c (persist-db/<release-access-handles repo)))
+     (state/add-repo! {:url repo})
      (op-mem-layer/update-local-tx! repo t))))
 
-
 (defn <download-graph
   [state repo graph-uuid]
   (go-try

src/main/frontend/db_worker.cljs

@@ -2,7 +2,6 @@
   "Worker used for browser DB implementation"
   (:require [promesa.core :as p]
             [datascript.storage :refer [IStorage]]
-            [cljs.cache :as cache]
             [clojure.edn :as edn]
             [datascript.core :as d]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
@@ -13,6 +12,7 @@
             [clojure.string :as string]
             [cljs-bean.core :as bean]
             [frontend.worker.search :as search]
+            [frontend.util :as util]
             [logseq.db.sqlite.util :as sqlite-util]))
 
 (defonce *sqlite (atom nil))
@@ -23,6 +23,14 @@
 ;; repo -> pool
 (defonce *opfs-pools (atom nil))
 
+(defn sanitize-db-name
+  [db-name]
+  (-> db-name
+      (string/replace " " "_")
+      (string/replace "/" "_")
+      (string/replace "\\" "_")
+      (string/replace ":" "_")))
+
 (defn- get-sqlite-conn
   [repo & {:keys [search?]
            :or {search? false}
@@ -41,7 +49,7 @@
 (defn- <get-opfs-pool
   [graph]
   (or (get-opfs-pool graph)
-      (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" graph)
+      (p/let [^js pool (.installOpfsSAHPoolVfs @*sqlite #js {:name (str "logseq-pool-" (sanitize-db-name graph))
                                                              :initialCapacity 20})]
         (swap! *opfs-pools assoc graph pool)
         pool)))
@@ -62,7 +70,7 @@
 
 (defn- get-repo-path
   [repo]
-  (str "/" repo ".sqlite"))
+  (str "/" (sanitize-db-name repo) ".sqlite"))
 
 (defn- <export-db-file
   [repo]
@@ -77,13 +85,17 @@
 
 (defn upsert-addr-content!
   "Upsert addr+data-seq"
-  [repo data]
+  [repo data delete-addrs]
   (let [^Object db (get-sqlite-conn repo)]
     (assert (some? db) "sqlite db not exists")
     (.transaction db (fn [tx]
                        (doseq [item data]
                          (.exec tx #js {:sql "INSERT INTO kvs (addr, content) values ($addr, $content) on conflict(addr) do update set content = $content"
-                                        :bind item}))))))
+                                        :bind item}))
+
+                       (doseq [addr delete-addrs]
+                         (.exec db #js {:sql "Delete from kvs where addr = ?"
+                                        :bind #js [addr]}))))))
 
 (defn restore-data-from-addr
   [repo addr]
@@ -96,28 +108,30 @@
       (edn/read-string content))))
 
 (defn new-sqlite-storage
-  [repo {:keys [threshold]
-         :or {threshold 4096}}]
-  (let [_cache (cache/lru-cache-factory {} :threshold threshold)]
-    (reify IStorage
-      (-store [_ addr+data-seq]
-        (let [data (map
-                    (fn [[addr data]]
-                      #js {:$addr addr
-                           :$content (pr-str data)})
-                    addr+data-seq)]
-          (upsert-addr-content! repo data)))
-
-      (-restore [_ addr]
-        (restore-data-from-addr repo addr)))))
+  [repo _opts]
+  (reify IStorage
+    (-store [_ addr+data-seq delete-addrs]
+      (util/profile
+       (str "SQLite store addr+data count: " (count addr+data-seq))
+       (let [data (map
+                   (fn [[addr data]]
+                     #js {:$addr addr
+                          :$content (pr-str data)})
+                   addr+data-seq)]
+         (upsert-addr-content! repo data delete-addrs))))
+
+    (-restore [_ addr]
+      (restore-data-from-addr repo addr))))
 
 (defn- close-db-aux!
   [repo ^Object db ^Object search]
   (swap! *sqlite-conns dissoc repo)
   (swap! *datascript-conns dissoc repo)
-  (swap! *opfs-pools dissoc repo)
   (when db (.close db))
-  (when search (.close search)))
+  (when search (.close search))
+  (when-let [^js pool (get-opfs-pool repo)]
+    (.releaseAccessHandles pool))
+  (swap! *opfs-pools dissoc repo))
 
 (defn- close-other-dbs!
   [repo]
@@ -133,7 +147,10 @@
 (defn- create-or-open-db!
   [repo]
   (when-not (get-sqlite-conn repo)
-    (p/let [pool (<get-opfs-pool repo)
+    (p/let [^js pool (<get-opfs-pool repo)
+            capacity (.getCapacity pool)
+            _ (when (zero? capacity)   ; file handle already releases since pool will be initialized only once
+                (.acquireAccessHandles pool))
             path (get-repo-path repo)
             db (new (.-OpfsSAHPoolDb pool) path)
             search-db (new (.-OpfsSAHPoolDb pool) (str "search-" path))
@@ -143,7 +160,6 @@
       (.exec db "PRAGMA locking_mode=exclusive")
       (sqlite-common-db/create-kvs-table! db)
       (search/create-tables-and-triggers! search-db)
-      (prn :debug :repo repo)
       (let [schema (sqlite-util/get-schema repo)
             conn (sqlite-common-db/get-storage-conn storage schema)]
         (swap! *datascript-conns assoc repo conn)
@@ -225,11 +241,11 @@
                           (string/replace-first (.-name file) ".logseq-pool-" "")))
                       all-files)
                 distinct)]
-     (prn :debug :all-files (map #(.-name %) all-files))
-     (prn :debug :all-files-count (count (filter
-                                          #(= (.-kind %) "file")
-                                          all-files)))
-     (prn :dbs dbs)
+     ;; (prn :debug :all-files (map #(.-name %) all-files))
+     ;; (prn :debug :all-files-count (count (filter
+     ;;                                      #(= (.-kind %) "file")
+     ;;                                      all-files)))
+     ;; (prn :dbs dbs)
      (bean/->js dbs)))
 
   (createOrOpenDB
@@ -237,17 +253,24 @@
    (p/let [_ (close-other-dbs! repo)]
      (create-or-open-db! repo)))
 
+  (getMaxTx
+   [_this repo]
+   (when-let [conn (get-datascript-conn repo)]
+     (:max-tx @conn)))
+
   (transact
    [_this repo tx-data tx-meta]
    (when-let [conn (get-datascript-conn repo)]
-     (try
-       (let [tx-data (edn/read-string tx-data)
-             tx-meta (edn/read-string tx-meta)]
-         (d/transact! conn tx-data tx-meta)
-         nil)
-       (catch :default e
-         (prn :debug :error)
-         (js/console.error e)))))
+     (util/profile
+      "DB transact!"
+      (try
+        (let [tx-data (edn/read-string tx-data)
+              tx-meta (edn/read-string tx-meta)]
+          (d/transact! conn tx-data tx-meta)
+          nil)
+        (catch :default e
+          (prn :debug :error)
+          (js/console.error e))))))
 
   (getInitialData
    [_this repo]
@@ -262,6 +285,11 @@
            _result (remove-vfs! pool)]
      nil))
 
+  (releaseAccessHandles
+   [_this repo]
+   (when-let [^js pool (get-opfs-pool repo)]
+     (.releaseAccessHandles pool)))
+
   (exportDB
    [_this repo]
    (<export-db-file repo))

src/main/frontend/handler.cljs

@@ -49,8 +49,7 @@
             [cljs-bean.core :as bean]
             [frontend.handler.test :as test]
             [frontend.db.rtc.op-mem-layer :as op-mem-layer]
-            [frontend.persist-db.browser :as db-browser]
-            [frontend.persist-db :as persist-db]))
+            [frontend.persist-db.browser :as db-browser]))
 
 (defn- set-global-error-notification!
   []
@@ -247,7 +246,6 @@
                _ (mobile-util/hide-splash) ;; hide splash as early as ui is stable
                repo (or (state/get-current-repo) (:url (first repos)))
                _ (restore-and-setup! repo repos)]
-         (persist-db/run-periodically-export!)
          (when (mobile-util/native-platform?)
            (state/restore-mobile-theme!)))
        (p/catch (fn [e]

src/main/frontend/handler/events.cljs

@@ -939,13 +939,23 @@
 (defmethod handle :editor/edit-block [[_ block pos id opts]]
   (editor-handler/edit-block! block pos id opts))
 
-(defmethod handle :db/multiple-tabs-opfs-failed [_]
-  ;; close current tab or window
-  (notification/show!
-   (let [word (if (util/electron?) "window" "tab")]
-     (util/format "Logseq doesn't support multiple %ss access to the same graph yet, please close this %s."
-                  word word))
-   :warning false))
+(rum/defc multi-tabs-dialog
+  []
+  (let [word (if (util/electron?) "window" "tab")]
+    [:div.flex.p-4.flex-col.gap-4.h-64
+     [:span.warning.text-lg
+      (util/format "Logseq doesn't support multiple %ss access to the same graph yet, please close this %s or switch to another graph."
+                   word word)]
+     [:div.text-lg
+      [:p "Switch to another repo: "]
+      (repo/repos-dropdown {:on-click (fn [e]
+                                        (util/stop e)
+                                        (state/set-state! :error/multiple-tabs-access-opfs? false)
+                                        (state/close-modal!))})]]))
+
+(defmethod handle :show/multiple-tabs-error-dialog [_]
+  (state/set-state! :error/multiple-tabs-access-opfs? true)
+  (state/set-modal! multi-tabs-dialog {:container-overflow-visible? true}))
 
 (defn run!
   []

src/main/frontend/handler/repo.cljs

@@ -22,6 +22,7 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.util.fs :as util-fs]
+            [frontend.util.text :as text-util]
             [frontend.persist-db :as persist-db]
             [promesa.core :as p]
             [shadow.resource :as rc]
@@ -345,12 +346,18 @@
         delete-db-f (fn []
                       (let [current-repo (state/get-current-repo)]
                         (db/remove-conn! url)
-                        (db-persist/delete-graph! url)
+                        (when db-based? (db-persist/delete-graph! url))
                         (search/remove-db! url)
                         (state/delete-repo! repo)
-                        (when (= current-repo url)
+                        (if (= current-repo url)
                           (when-let [graph (:url (first (state/get-repos)))]
-                            (state/pub-event! [:graph/switch graph {:persist? false}])))))]
+                            (notification/show! (str "Removed graph "
+                                                     (pr-str (text-util/get-graph-name-from-path url))
+                                                     ". Redirecting to graph "
+                                                     (pr-str (text-util/get-graph-name-from-path url)))
+                                                :success)
+                            (state/pub-event! [:graph/switch graph {:persist? false}]))
+                          (notification/show! (str "Removed graph " (pr-str (text-util/get-graph-name-from-path url))) :success))))]
     (when (or (config/local-file-based-graph? url)
               db-based?
               (config/demo-graph? url))

src/main/frontend/modules/outliner/pipeline.cljs

@@ -50,8 +50,8 @@
                   (state/set-edit-content! input content)
                   (when pos (cursor/move-cursor-to input pos)))))))))))
 
-(defn- delete-property-parent-block-if-empty!
-  [repo tx-report deleted-block-uuids]
+(defn- delete-property-parent-block-if-empty
+  [tx-report deleted-block-uuids]
   (let [empty-property-parents (->> (keep (fn [child-id]
                                             (let [e (d/entity (:db-before tx-report) [:block/uuid child-id])]
                                               (when (:created-from-property (:block/metadata (:block/parent e)))
@@ -60,23 +60,21 @@
                                                     parent-now))))) deleted-block-uuids)
                                     distinct)]
     (when (seq empty-property-parents)
-      (let [tx-data (->>
-                     (mapcat (fn [b]
-                               (let [{:keys [created-from-block created-from-property]} (:block/metadata b)
-                                     created-block (db/entity [:block/uuid created-from-block])
-                                     properties (assoc (:block/properties created-block) created-from-property "")]
-                                 (when (and created-block created-from-property)
-                                   [[:db/retractEntity (:db/id b)]
-                                    [:db/add (:db/id created-block) :block/properties properties]])))
-                             empty-property-parents)
-                     (remove nil?))]
-        (db/transact! repo tx-data {:outliner/transact? true
-                                    :replace? true})))))
+      (->>
+       (mapcat (fn [b]
+                 (let [{:keys [created-from-block created-from-property]} (:block/metadata b)
+                       created-block (db/entity [:block/uuid created-from-block])
+                       properties (assoc (:block/properties created-block) created-from-property "")]
+                   (when (and created-block created-from-property)
+                     [[:db/retractEntity (:db/id b)]
+                      [:db/add (:db/id created-block) :block/properties properties]])))
+               empty-property-parents)
+       (remove nil?)))))
 
 (defn invoke-hooks
   [tx-report]
   (let [tx-meta (:tx-meta tx-report)
-        {:keys [compute-path-refs? from-disk? new-graph? replace?]} tx-meta]
+        {:keys [from-disk? new-graph? pipeline-replace?]} tx-meta]
     (when (and (not from-disk?)
                (not new-graph?))
       (try
@@ -87,51 +85,46 @@
 
       (let [{:keys [pages blocks]} (ds-report/get-blocks-and-pages tx-report)
             repo (state/get-current-repo)
-            tx (when-not compute-path-refs?
-                 (util/profile
-                  "Compute path refs: "
-                  (set (compute-block-path-refs-tx tx-report blocks))))
-            tx-report' (if (seq tx)
-                         (let [refs-tx-data' (:tx-data (db/transact! repo tx {:outliner/transact? true
-                                                                              :replace? true
-                                                                              :compute-path-refs? true}))]
-                           ;; merge
-                           (assoc tx-report :tx-data (concat (:tx-data tx-report) refs-tx-data')))
-                         tx-report)
             importing? (:graph/importing @state/state)
-            deleted-block-uuids (set (outliner-pipeline/filter-deleted-blocks (:tx-data tx-report)))]
+            deleted-block-uuids (set (outliner-pipeline/filter-deleted-blocks (:tx-data tx-report)))
+            replace-full-tx (when-not pipeline-replace?
+                              (concat
+                                 ;; block path refs
+                               (util/profile
+                                "Compute path refs: "
+                                (set (compute-block-path-refs-tx tx-report blocks)))
 
-        (when (and (seq deleted-block-uuids) (not replace?)
-                   (not compute-path-refs?))
-          (delete-property-parent-block-if-empty! repo tx-report deleted-block-uuids))
+                                 ;; delete empty property parent block
+                               (when (seq deleted-block-uuids)
+                                 (delete-property-parent-block-if-empty tx-report deleted-block-uuids))
 
-        (let [updated-blocks (remove (fn [b] (contains? (set deleted-block-uuids)  (:block/uuid b))) blocks)
-              tx-id (get-in tx-report' [:tempids :db/current-tx])
-              update-tx-ids (->>
-                             (map (fn [b]
-                                    (when-let [db-id (:db/id b)]
-                                      {:db/id db-id
-                                       :block/tx-id tx-id})) updated-blocks)
-                             (remove nil?))]
-          (when (and (seq update-tx-ids)
-                     (not (:update-tx-ids? tx-meta)))
-            (db/transact! repo update-tx-ids {:replace? true
-                                              :update-tx-ids? true}))
-          (when (not config/publishing?)
-            (persist-db/<transact-data repo (:tx-data tx-report) (:tx-meta tx-report))))
+                                 ;; update block/tx-id
+                               (let [updated-blocks (remove (fn [b] (contains? (set deleted-block-uuids)  (:block/uuid b))) blocks)
+                                     tx-id (get-in tx-report [:tempids :db/current-tx])]
+                                 (->>
+                                  (map (fn [b]
+                                         (when-let [db-id (:db/id b)]
+                                           {:db/id db-id
+                                            :block/tx-id tx-id})) updated-blocks)
+                                  (remove nil?)))))]
 
-        (when-not importing?
-          (react/refresh! repo tx-report'))
+        (when (and (not config/publishing?) (not pipeline-replace?))
+          (let [tx-report' (db/transact! repo replace-full-tx {:replace? true
+                                                               :pipeline-replace? true})
+                full-tx-data (concat (:tx-data tx-report) (:tx-data tx-report'))]
+            (persist-db/<transact-data repo full-tx-data (:tx-meta tx-report))
+            (when-not importing?
+              (react/refresh! repo (assoc tx-report :tx-data full-tx-data)))))
 
         (when (and (not (:delete-files? tx-meta))
-                   (not replace?))
+                   (not pipeline-replace?))
           (doseq [p (seq pages)]
             (updated-page-hook tx-report p)))
 
         (when (and state/lsp-enabled?
                    (seq blocks)
                    (not importing?)
-                   (not replace?)
+                   (not pipeline-replace?)
                    (<= (count blocks) 1000))
           (state/pub-event! [:plugin/hook-db-tx
                              {:blocks  blocks

src/main/frontend/persist_db.cljs

@@ -1,11 +1,9 @@
 (ns frontend.persist-db
-   "Backend of DB based graph"
-   (:require [frontend.persist-db.browser :as browser]
-             [frontend.persist-db.protocol :as protocol]
-             [promesa.core :as p]
-             [frontend.config :as config]
-             [frontend.state :as state]
-             [frontend.util :as util]))
+  "Backend of DB based graph"
+  (:require [frontend.persist-db.browser :as browser]
+            [frontend.persist-db.protocol :as protocol]
+            [promesa.core :as p]
+            [electron.ipc :as ipc]))
 
 (defonce opfs-db (browser/->InBrowser))
 
@@ -35,25 +33,30 @@
   ([repo]
    (<fetch-init-data repo {}))
   ([repo opts]
-   (p/let [ret (protocol/<fetch-initial-data (get-impl) repo opts)]
-     (js/console.log "fetch-initial-data" ret)
-     ret)))
+   (p/do!
+    (ipc/ipc :db-open repo)
+    (protocol/<fetch-initial-data (get-impl) repo opts))))
 
 ;; FIXME: limit repo name's length and sanity
-;; original size is 56
 ;; @shuyu Do we still need this?
 (defn <new [repo]
-  {:pre [(<= (count repo) 128)]}
-  (p/let [_ (protocol/<new (get-impl) repo)]
-    (<export-db repo {})))
-
-(defn run-periodically-export!
-  []
-  (js/setInterval
-   (fn []
-     (when-let [repo (state/get-current-repo)]
-       (when (and (util/electron?) (config/db-based-graph? repo))
-         (println :debug :save-db-to-disk repo)
-         (<export-db repo {}))))
+  {:pre [(<= (count repo) 56)]}
+  (p/let [_ (protocol/<new (get-impl) repo)
+          _ (<export-db repo {})]
+    (ipc/ipc :db-open repo)))
+
+(defn <release-access-handles
+  [repo]
+  (protocol/<release-access-handles (get-impl) repo))
+
+(comment
+  (defn run-export-periodically!
+    []
+    (js/setInterval
+     (fn []
+       (when-let [repo (state/get-current-repo)]
+         (when (and (util/electron?) (config/db-based-graph? repo))
+           (println :debug :save-db-to-disk repo)
+           (<export-db repo {}))))
    ;; every 10 minutes
-   (* 10 60 1000)))
+     (* 10 60 1000))))

src/main/frontend/persist_db/browser.cljs

@@ -40,7 +40,7 @@
 (defn <export-db!
   [repo data]
   (cond
-    (and (util/electron?) (config/db-based-graph? repo))
+    (util/electron?)
     (ipc/ipc :db-export repo data)
 
     ;; TODO: browser nfs-supported? auto backup
@@ -49,6 +49,12 @@
     :else
     nil))
 
+(defn- sqlite-error-handler
+  [error]
+  (if (= "NoModificationAllowedError"  (.-name error))
+    (state/pub-event! [:show/multiple-tabs-error-dialog])
+    (notification/show! [:div (str "SQLiteDB error: " error)] :error)))
+
 (defrecord InBrowser []
   protocol/PersistentDB
   (<new [_this repo]
@@ -60,30 +66,30 @@
       (-> (.listDB sqlite)
           (p/then (fn [result]
                     (bean/->clj result)))
-          (p/catch (fn [error]
-                     (prn :debug :list-db-error (js/Date.))
-                     (if (= "NoModificationAllowedError"  (.-name error))
-                       (state/pub-event! [:db/multiple-tabs-opfs-failed])
-                       (notification/show! [:div (str "SQLiteDB error: " error)] :error))
-                     [])))))
+          (p/catch sqlite-error-handler))))
 
   (<unsafe-delete [_this repo]
     (when-let [^js sqlite @*sqlite]
       (.unsafeUnlinkDB sqlite repo)))
 
+  (<release-access-handles [_this repo]
+    (when-let [^js sqlite @*sqlite]
+      (.releaseAccessHandles sqlite repo)))
+
   (<transact-data [_this repo tx-data tx-meta]
-    (let [^js sqlite @*sqlite]
-      (p/let [_ (when sqlite (.transact sqlite repo (pr-str tx-data) (pr-str tx-meta)))]
-        nil)))
+    (let [^js sqlite @*sqlite
+          tx-data' (pr-str tx-data)
+          tx-meta' (pr-str tx-meta)]
+      (p/do!
+       (ipc/ipc :db-transact repo tx-data' tx-meta')
+       (when sqlite (.transact sqlite repo tx-data' tx-meta'))
+       nil)))
 
   (<fetch-initial-data [_this repo _opts]
     (when-let [^js sqlite @*sqlite]
       (-> (p/let [_ (.createOrOpenDB sqlite repo)]
             (.getInitialData sqlite repo))
-          (p/catch (fn [error]
-                     (prn :debug :fetch-initial-data-error repo)
-                     (js/console.error error)
-                     (notification/show! [:div (str "SQLiteDB fetch error: " error)] :error) {})))))
+          (p/catch sqlite-error-handler))))
 
   (<export-db [_this repo opts]
     (when-let [^js sqlite @*sqlite]

src/main/frontend/persist_db/protocol.cljs

@@ -5,6 +5,7 @@
   (<list-db [this] "List all databases")
   (<new [this repo] "Create or open a graph")
   (<unsafe-delete [this repo] "Delete graph and its vfs")
+  (<release-access-handles [this repo] "Release access file handles")
   (<transact-data [this repo tx-data tx-meta] "Transact data to db")
   (<fetch-initial-data [this repo opts] "Fetch Initial data")
   (<export-db [this repo opts] "Save or get SQLite db")

src/main/frontend/state.cljs

@@ -1529,7 +1529,7 @@ Similar to re-frame subscriptions"
 
 (defn close-modal!
   []
-  (when-not (editing?)
+  (when-not (or (editing?) (:error/multiple-tabs-access-opfs? @state))
     (close-dropdowns!)
     (if (seq (get-sub-modals))
       (close-sub-modal!)

yarn.lock

@@ -552,10 +552,10 @@
   resolved "https://registry.yarnpkg.com/@logseq/react-tweet-embed/-/react-tweet-embed-1.3.1-1.tgz#119d22be8234de006fc35c3fa2a36f85634c5be6"
   integrity sha512-9O0oHs5depCvh6ZQvwtl1xb7B80YG5rUfY10uSUat5itOlcE3IWaYYpe6p/tcHErqHWnWgkXHitAB9M29FMbQg==
 
-"@logseq/sqlite-wasm@=0.0.7":
-  version "0.0.7"
-  resolved "https://registry.yarnpkg.com/@logseq/sqlite-wasm/-/sqlite-wasm-0.0.7.tgz#c887e2e39dcf2b08b1fa4b2730bf4a9c3d8ae0ac"
-  integrity sha512-8bbAx4FW5p8JNNPjjwAMW9ZGfI8OzEmamBD6ZMupmVzHjYdHXvwG52Upinkd28p2pzmw5tpMBip9/3zVKgH6Pg==
+"@logseq/sqlite-wasm@=0.1.0":
+  version "0.1.0"
+  resolved "https://registry.yarnpkg.com/@logseq/sqlite-wasm/-/sqlite-wasm-0.1.0.tgz#b866d22d7a83bfa6815fe7f4d8899097b49af642"
+  integrity sha512-Ft/ZY8b2s35pFS7L7NX0CKWa5WkeKueYB7l5AYFFvQ2QQ0s7O4zlEi+jIX/ZdO2vnf2p4ALg+QvgaPgu6w2o5w==
 
 "@mapbox/node-pre-gyp@^1.0.0":
   version "1.0.11"