Use sqlite.db ns in electron.db

- Add more tests for sqlite.db
- Update readme and CI for db's tests

.clj-kondo/config.edn

@@ -107,6 +107,7 @@
              logseq.common.path path
              logseq.common.graph common-graph
              logseq.common.config common-config
+             logseq.db.sqlite.db sqlite-db
              logseq.graph-parser graph-parser
              logseq.graph-parser.text text
              logseq.graph-parser.block gp-block

.github/workflows/db.yml

@@ -56,6 +56,9 @@ jobs:
       - name: Fetch yarn deps
         run: yarn install --frozen-lockfile
 
+      - name: Run nbb-logseq tests
+        run: yarn test
+
       # In this job because it depends on an npm package
       - name: Load namespaces into nbb-logseq
         run: bb test:load-all-namespaces-with-nbb .

deps/db/.carve/config.edn

@@ -1,6 +1,6 @@
 {:paths ["src"]
- :api-namespaces [
+ :api-namespaces [logseq.db.sqlite.db
+                  logseq.db.sqlite.util
                   ;; Some fns are used by frontend but not worth moving over yet
-                  logseq.db.schema
-                  ]
+                  logseq.db.schema]
  :report {:format :ignore}}

deps/db/README.md

@@ -1,15 +1,16 @@
 ## Description
 
 This library provides a minimal API for using a
-[datascript](https://github.com/tonsky/datascript) database from the Logseq app
-and the CLI. This library is compatible with ClojureScript and with
+[datascript](https://github.com/tonsky/datascript) or
+[SQLite](https://www.sqlite.org/index.html) database from the Logseq app and the
+CLI. This library is compatible with ClojureScript and with
 [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
 frontend and commandline functionality.
 
 ## API
 
 This library is under the parent namespace `logseq.db`. This library provides
-two main namespaces, `logseq.db` and `logseq.db.rules`.
+three main namespaces, `logseq.db`, `logseq.db.rules` and `logseq.db.sqlite.db`.
 
 ## Usage
 
@@ -24,23 +25,34 @@ file](/.github/workflows/db.yml) for linting examples.
 
 ### Setup
 
-To run linters, you'll want to install yarn dependencies once:
+To run linters and tests, you'll want to install yarn dependencies once:
 ```
 yarn install
 ```
 
 This step is not needed if you're just running the application.
 
-## Linting
+### Testing
 
+Testing is done with nbb-logseq and
+[nbb-test-runner](https://github.com/nextjournal/nbb-test-runner). Some basic
+usage:
+
+```
+# Run all tests
+$ yarn test
+# List available options
+$ yarn test -H
+# Run tests with :focus metadata flag
+$ yarn test -i focus
+```
 ### Datalog linting
 
-Our rules are linted through a script that also uses the datalog-parser. To run this linter:
+Datalog rules for the client are linted through a script that also uses the datalog-parser. To run this linter:
 ```
 bb lint:rules
 ```
 
-
 ### Managing dependencies
 
 The package.json dependencies are just for testing and should be updated if there is

deps/db/src/logseq/db/sqlite/db.cljs

@@ -58,14 +58,12 @@
     (let [create-index-stmt (prepare db "CREATE INDEX IF NOT EXISTS block_type ON blocks(type)" db-name)]
       (.run ^object create-index-stmt))))
 
-;; TODO: Wrap in electron with (fs/ensureDirSync graph-dir)
 (defn get-db-full-path
   [graphs-dir db-name]
   (let [db-name' (sanitize-db-name db-name)
         graph-dir (node-path/join graphs-dir db-name')]
     [db-name' (node-path/join graph-dir "db.sqlite")]))
 
-;; TODO: Wrap in electron with try
 (defn open-db!
   [graphs-dir db-name]
   (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
@@ -83,18 +81,17 @@
                 (string/join ", ")) ")"))
 
 (defn upsert-blocks!
-  [graphs-dir repo blocks]
-  (if-let [db (get-db repo)]
+  "Creates or updates given js blocks. Returns true if transaction is successful"
+  [repo blocks]
+  (when-let [db (get-db repo)]
     (let [insert (prepare db "INSERT INTO blocks (uuid, type, page_uuid, page_journal_day, name, content,datoms, created_at, updated_at) VALUES (@uuid, @type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at) ON CONFLICT (uuid) DO UPDATE SET (type, page_uuid, page_journal_day, name, content, datoms, created_at, updated_at) = (@type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at)"
                           repo)
           insert-many (.transaction ^object db
                                     (fn [blocks]
                                       (doseq [block blocks]
                                         (.run ^object insert block))))]
-      (insert-many blocks))
-    (do
-      (open-db! graphs-dir repo)
-      (upsert-blocks! graphs-dir repo blocks))))
+      (insert-many blocks)
+      true)))
 
 (defn delete-blocks!
   [repo uuids]

deps/db/test/logseq/db/sqlite/db_test.cljs

@@ -21,6 +21,25 @@
   [dir db-name]
   (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true}))
 
+(deftest get-initial-data
+  (testing "Fetches file block"
+    (create-graph-dir "tmp/graphs" "test-db")
+    (sqlite-db/open-db! "tmp/graphs" "test-db")
+    (let [blocks (mapv sqlite-util/ds->sqlite-block
+                       [{:block/uuid (random-uuid)
+                         :file/path "logseq/config.edn"
+                         :file/content "{:foo :bar}"}])
+          _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))]
+      (is (= {:content "{:foo :bar}"
+              :name nil
+              :type 3}
+             (-> (sqlite-db/get-initial-data "test-db")
+                 :init-data
+                 bean/->clj
+                 first
+                 (select-keys [:content :name :type])))
+          "Correct file with content is found"))))
+
 (deftest upsert-blocks!
   (let [page-uuid (random-uuid)
         block-uuid (random-uuid)
@@ -28,7 +47,7 @@
     (create-graph-dir "tmp/graphs" "test-db")
     (sqlite-db/open-db! "tmp/graphs" "test-db")
 
-    (testing "create a journal block"
+    (testing "Creates a journal block"
       (let [blocks (mapv sqlite-util/ds->sqlite-block
                          [{:block/uuid page-uuid
                            :block/journal-day 20230629
@@ -41,7 +60,7 @@
                            :block/created-at created-at
                            :block/updated-at created-at
                            :page_uuid page-uuid}])
-            _ (sqlite-db/upsert-blocks! "tmp/graphs" "test-db" (bean/->js blocks))
+            _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
             db-data (sqlite-db/get-initial-data "test-db")]
         (is (= {:uuid (str page-uuid) :page_journal_day 20230629
                 :name "jun 29th, 2023" :type 2
@@ -67,7 +86,7 @@
                (-> db-data :all-blocks bean/->clj))
             "Correct block and page uuid pairs exist")))
 
-    (testing "update a block"
+    (testing "Updates a block"
       (let [updated-at 1688072416134
             blocks (mapv sqlite-util/ds->sqlite-block
                          [{:block/uuid page-uuid
@@ -81,7 +100,7 @@
                            :block/created-at created-at
                            :block/updated-at updated-at
                            :page_uuid page-uuid}])
-            _ (sqlite-db/upsert-blocks! "tmp/graphs" "test-db" (bean/->js blocks))
+            _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
             db-data (sqlite-db/get-initial-data "test-db")]
         (is (= {:uuid (str page-uuid) :updated_at updated-at :created_at created-at}
                (-> db-data
@@ -97,4 +116,26 @@
                    first
                    bean/->clj
                    (select-keys [:content :created_at :updated_at])))
-            "Updated block has correct content and timestamps")))))
+            "Updated block has correct content and timestamps")))))
+
+(deftest get-other-data
+  (testing "Retrieves a normal page block"
+    (create-graph-dir "tmp/graphs" "test-db")
+    (sqlite-db/open-db! "tmp/graphs" "test-db")
+    (let [page-uuid (random-uuid)
+          block-uuid (random-uuid)
+          blocks (mapv sqlite-util/ds->sqlite-block
+                       [{:block/uuid page-uuid
+                         :block/name "some page"}
+                        {:block/content "test"
+                         :block/uuid block-uuid
+                         :block/page {:db/id 100022}
+                         :page_uuid page-uuid}])]
+      (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
+      (is (= {:content "test" :uuid (str block-uuid)
+              :page_uuid (str page-uuid) :type 1}
+             (-> (sqlite-db/get-other-data "test-db" [])
+                 bean/->clj
+                 first
+                 (select-keys [:content :page_uuid :type :uuid])))
+          "New page block is fetched with get-other-data"))))

src/electron/electron/db.cljs

@@ -1,62 +1,12 @@
 (ns electron.db
-  "SQLite db"
+  "Provides SQLite dbs for electron and manages files of those dbs"
   (:require ["path" :as node-path]
             ["fs-extra" :as fs]
-            ["better-sqlite3" :as sqlite3]
-            [clojure.string :as string]
             ["electron" :refer [app]]
             [electron.logger :as logger]
-            [cljs-bean.core :as bean]))
+            [logseq.db.sqlite.db :as sqlite-db]))
 
-;; use built-in blocks to represent db schema, config, custom css, custom js, etc.
-
-(defonce databases (atom nil))
-
-(defn close!
-  []
-  (when @databases
-    (doseq [[_ database] @databases]
-      (.close database))
-    (reset! databases nil)))
-
-(defn sanitize-db-name
-  [db-name]
-  (-> db-name
-      (string/replace "logseq_db_" "")
-      (string/replace "/" "_")
-      (string/replace "\\" "_")
-      (string/replace ":" "_"))) ;; windows
-
-(defn get-db
-  [repo]
-  (get @databases (sanitize-db-name repo)))
-
-(defn prepare
-  [^object db sql db-name]
-  (when db
-    (try
-      (.prepare db sql)
-      (catch :default e
-        (logger/error (str "SQLite prepare failed: " e ": " db-name))
-        (throw e)))))
-
-(defn create-blocks-table!
-  [db db-name]
-  (let [stmt (prepare db "CREATE TABLE IF NOT EXISTS blocks (
-                        uuid TEXT PRIMARY KEY,
-                        type INTEGER,
-                        page_uuid TEXT,
-                        page_journal_day INTEGER,
-                        name TEXT,
-                        content TEXT,
-                        datoms TEXT,
-                        created_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL,
-                        updated_at DATETIME DEFAULT CURRENT_TIMESTAMP NOT NULL
-                        )"
-                      db-name)]
-    (.run ^object stmt)
-    (let [create-index-stmt (prepare db "CREATE INDEX IF NOT EXISTS block_type ON blocks(type)" db-name)]
-      (.run ^object create-index-stmt))))
+(def close! sqlite-db/close!)
 
 ;; ~/logseq
 (defn get-graphs-dir
@@ -68,94 +18,26 @@
   []
   (fs/ensureDirSync (get-graphs-dir)))
 
-(defn get-db-full-path
-  [db-name]
-  (let [db-name' (sanitize-db-name db-name)
-        dir (get-graphs-dir)
-        graph-dir (node-path/join dir db-name')]
-    (fs/ensureDirSync graph-dir)
-    [db-name' (node-path/join graph-dir "db.sqlite")]))
-
 (defn open-db!
   [db-name]
-  (let [[db-sanitized-name db-full-path] (get-db-full-path db-name)]
-    (try (let [db (sqlite3 db-full-path nil)]
-           (create-blocks-table! db db-name)
-           (swap! databases assoc db-sanitized-name db))
+  (let [graphs-dir (get-graphs-dir)]
+    (fs/ensureDirSync (node-path/join graphs-dir (sqlite-db/sanitize-db-name db-name)))
+    (try (sqlite-db/open-db! graphs-dir db-name)
          (catch :default e
            (logger/error (str e ": " db-name))
            ;; (fs/unlinkSync db-full-path)
            ))))
 
-(defn- clj-list->sql
-  "Turn clojure list into SQL list
-   '(1 2 3 4)
-   ->
-   \"('1','2','3','4')\""
-  [ids]
-  (str "(" (->> (map (fn [id] (str "'" id "'")) ids)
-                (string/join ", ")) ")"))
-
 (defn upsert-blocks!
   [repo blocks]
-  (if-let [db (get-db repo)]
-    (let [insert (prepare db "INSERT INTO blocks (uuid, type, page_uuid, page_journal_day, name, content,datoms, created_at, updated_at) VALUES (@uuid, @type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at) ON CONFLICT (uuid) DO UPDATE SET (type, page_uuid, page_journal_day, name, content, datoms, created_at, updated_at) = (@type, @page_uuid, @page_journal_day, @name, @content, @datoms, @created_at, @updated_at)"
-                          repo)
-          insert-many (.transaction ^object db
-                                    (fn [blocks]
-                                      (doseq [block blocks]
-                                        (.run ^object insert block))))]
-      (insert-many blocks))
-    (do
-      (open-db! repo)
-      (upsert-blocks! repo blocks))))
-
-(defn delete-blocks!
-  [repo uuids]
-  (when-let [db (get-db repo)]
-    (let [sql (str "DELETE from blocks WHERE uuid IN " (clj-list->sql uuids))
-          stmt (prepare db sql repo)]
-      (.run ^object stmt))))
-
-
-;; Initial data:
-;; All pages and block ids
-;; latest 3 journals
-;; other data such as config.edn, custom css/js
-;; current page, sidebar blocks
-
-(defn- query
-  [repo db sql]
-  (let [stmt (prepare db sql repo)]
-    (.all ^object stmt)))
-
-(defn get-initial-data
-  [repo]
-  (when-let [db (get-db repo)]
-    (let [all-pages (query repo db "select * from blocks where type = 2") ; 2 = page block
-          ;; 1 = normal block
-          all-block-ids (query repo db "select uuid, page_uuid from blocks where type = 1")
-          recent-journal (some-> (query repo db "select uuid from blocks where type = 2 order by page_journal_day desc limit 1")
-                                 first
-                                 bean/->clj
-                                 :uuid)
-          latest-journal-blocks (when recent-journal
-                                  (query repo db (str "select * from blocks where type = 1 and page_uuid = '" recent-journal "'")))
-          init-data (query repo db "select * from blocks where type in (3, 4, 5, 6)")]
-      {:all-pages all-pages
-       :all-blocks all-block-ids
-       :journal-blocks latest-journal-blocks
-       :init-data init-data})))
-
-(defn get-other-data
-  [repo journal-block-uuids]
-  (when-let [db (get-db repo)]
-    (query repo db (str "select * from blocks where type = 1 and uuid not in "
-                        (clj-list->sql journal-block-uuids)))))
+  (or (sqlite-db/upsert-blocks! repo blocks)
+      ;; FIXME: When would an upsert not have a database connection?
+      (do (open-db! repo)
+          (sqlite-db/upsert-blocks! repo blocks))))
 
 (defn unlink-graph!
   [repo]
-  (let [db-name (sanitize-db-name repo)
+  (let [db-name (sqlite-db/sanitize-db-name repo)
         path (node-path/join (get-graphs-dir) db-name)
         unlinked (node-path/join (get-graphs-dir) "Unlinked graphs")
         new-path (node-path/join unlinked db-name)]

src/electron/electron/handler.cljs

@@ -29,6 +29,8 @@
             [electron.state :as state]
             [electron.utils :as utils]
             [electron.window :as win]
+            [logseq.db.sqlite.db :as sqlite-db]
+            [logseq.db.sqlite.util :as sqlite-util]
             [logseq.common.graph :as common-graph]
             [promesa.core :as p]))
 
@@ -365,55 +367,21 @@
 (defmethod handle :db-new [_window [_ repo]]
   (db/open-db! repo))
 
-(defn- type-of-block
-  "
-  TODO: use :block/type
-  | value | meaning                                        |
-  |-------+------------------------------------------------|
-  |     1 | normal block                                   |
-  |     2 | page block                                     |
-  |     3 | init data, (config.edn, custom.js, custom.css) |
-  |     4 | db schema                                      |
-  |     5 | unknown type                                   |
-  |     6 | property block                                 |
-  "
-  [block]
-  (cond
-    (:block/page block) 1
-    (:file/content block) 3
-    (= "property" (:block/type block)) 6
-    (:block/name block) 2
-    :else 5))
-
 (defmethod handle :db-transact-data [_window [_ repo data-str]]
   (let [data (reader/read-string data-str)
         {:keys [blocks deleted-block-uuids]} data]
     (when (seq deleted-block-uuids)
-      (db/delete-blocks! repo deleted-block-uuids))
+      (sqlite-db/delete-blocks! repo deleted-block-uuids))
     (when (seq blocks)
-      (let [blocks' (mapv
-                     (fn [b]
-                       {:uuid (str (:block/uuid b))
-                        :type (type-of-block b)
-                        :page_uuid (str (:page_uuid b))
-                        :page_journal_day (:block/journal-day b)
-                        :name (:block/name b)
-                        :content (or (:file/content b) (:block/content b))
-                        :datoms (:datoms b)
-                        :created_at (or (:block/created-at b) (utils/time-ms))
-                        :updated_at (or (:block/updated-at b) (utils/time-ms))})
-                     blocks)]
-        (prn :BLOCKS blocks)
-        (prn :BLOCKS' blocks')
-
+      (let [blocks' (mapv sqlite-util/ds->sqlite-block blocks)]
         (db/upsert-blocks! repo (bean/->js blocks'))))))
 
 (defmethod handle :get-initial-data [_window [_ repo _opts]]
   (db/open-db! repo)
-  (db/get-initial-data repo))
+  (sqlite-db/get-initial-data repo))
 
 (defmethod handle :get-other-data [_window [_ repo journal-block-uuids _opts]]
-  (db/get-other-data repo journal-block-uuids))
+  (sqlite-db/get-other-data repo journal-block-uuids))
 
 ;; DB related IPCs End