chore: mv logseq.db.sqlite.db into sqlite.cli

for more explicit name and purpose. Also updated outdated db readme
and removed unused fns in logseq.db.sqlite.db

.clj-kondo/config.edn

@@ -23,7 +23,8 @@
                                 object]}
 
   :discouraged-namespace
-  {logseq.db.sqlite.db {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}}
+  {logseq.db.sqlite.cli {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}
+   logseq.outliner.cli {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}}
 
   :unused-namespace {:level :warning
                      :exclude [logseq.db.frontend.entity-plus]}
@@ -154,7 +155,7 @@
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate
-             logseq.db.sqlite.db sqlite-db
+             logseq.db.sqlite.cli sqlite-cli
              logseq.db.sqlite.util sqlite-util
              logseq.graph-parser graph-parser
              logseq.graph-parser.text text

deps/db/.carve/config.edn

@@ -1,6 +1,5 @@
 {:paths ["src"]
- :api-namespaces [logseq.db.sqlite.db
-                  logseq.db.sqlite.common-db
+ :api-namespaces [logseq.db.sqlite.common-db
                   logseq.db.sqlite.rtc
                   logseq.db.sqlite.util
                   logseq.db.sqlite.cli

deps/db/.clj-kondo/config.edn

@@ -19,7 +19,7 @@
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate
-             logseq.db.sqlite.db sqlite-db
+             logseq.db.sqlite.cli sqlite-cli
              logseq.db.sqlite.util sqlite-util}}}
  :skip-comments true
  :output {:progress true}}

deps/db/README.md

@@ -1,16 +1,19 @@
 ## Description
 
-This library provides a minimal API to access the
+This library provides an API to the
 frontend([datascript](https://github.com/tonsky/datascript)) and
 backend([SQLite](https://www.sqlite.org/index.html)) databases from the Logseq
-app and the CLI. This library is compatible with ClojureScript and with
+app and the CLI. The majority of this library is focused on supporting DB graphs
+but there are a few older namespaces that support file graphs. This library is
+compatible with ClojureScript and with
 [nbb-logseq](https://github.com/logseq/nbb-logseq) to respectively provide
 frontend and commandline functionality.
 
 ## API
 
-This library is under the parent namespace `logseq.db`. This library provides
-three main namespaces, `logseq.db`, `logseq.db.frontend.rules` and `logseq.db.sqlite.db`.
+This library is under the parent namespace `logseq.db`. While `logseq.db` is the
+main entry point, this library also provides frontend namespaces under
+`logseq.db.frontend` and backend/sqlite namespaces under `logseq.db.sqlite`.
 
 ## Usage
 

deps/db/script/dump_datoms.cljs

@@ -4,7 +4,7 @@
      $ yarn -s nbb-logseq script/dump_datoms.cljs db-name datoms.edn"
     (:require [datascript.core :as d]
               [clojure.pprint :as pprint]
-              [logseq.db.sqlite.db :as sqlite-db]
+              [logseq.db.sqlite.cli :as sqlite-cli]
               [nbb.core :as nbb]
               ["path" :as path]
               ["os" :as os]
@@ -14,7 +14,7 @@
   "The db graph bare version of gp-cli/parse-graph"
   [graph-name]
   (let [graphs-dir (path/join (os/homedir) "logseq/graphs")]
-    (sqlite-db/open-db! graphs-dir graph-name)))
+    (sqlite-cli/open-db! graphs-dir graph-name)))
 
 (defn -main [args]
   (when (< (count args) 2)

deps/db/script/query.cljs

@@ -4,7 +4,7 @@
   $ yarn -s nbb-logseq script/query.cljs db-name '[:find (pull ?b [:block/name :block/content]) :where [?b :block/created-at]]'"
   (:require [datascript.core :as d]
             [clojure.edn :as edn]
-            [logseq.db.sqlite.db :as sqlite-db]
+            [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db.frontend.rules :as rules]
             [nbb.core :as nbb]
             [clojure.string :as string]
@@ -51,7 +51,7 @@
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
         [dir db-name] (get-dir-and-db-name graph-dir)
-        conn (sqlite-db/open-db! dir db-name)
+        conn (sqlite-cli/open-db! dir db-name)
         results (if (:entity options)
                   (map #(when-let [ent (d/entity @conn
                                                  (if (string? %) (edn/read-string %) %))]

deps/db/script/validate_client_db.cljs

@@ -1,7 +1,7 @@
 (ns validate-client-db
   "Script that validates the datascript db of a DB graph
    NOTE: This script is also used in CI to confirm our db's schema is up to date"
-  (:require [logseq.db.sqlite.db :as sqlite-db]
+  (:require [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.frontend.property :as db-property]
@@ -71,7 +71,7 @@
                               (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
                           ((juxt node-path/dirname node-path/basename) graph-dir'))
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (try (sqlite-db/open-db! dir db-name)
+        conn (try (sqlite-cli/open-db! dir db-name)
                   (catch :default e
                     (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))
                     (js/process.exit 1)))

deps/db/src/logseq/db/sqlite/cli.cljs

@@ -1,9 +1,89 @@
 (ns ^:node-only logseq.db.sqlite.cli
   "Primary ns to interact with DB graphs with node.js based CLIs"
-  (:require ["fs" :as fs]
+  (:require ["better-sqlite3" :as sqlite3]
+            [logseq.db.sqlite.common-db :as sqlite-common-db]
+            ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
+            #_:clj-kondo/ignore
+            [datascript.core :as d]
+            [datascript.storage :refer [IStorage]]
+            [goog.object :as gobj]
+            [clojure.edn :as edn]
+            [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.sqlite.util :as sqlite-util]
+            ["fs" :as fs]
             ["path" :as node-path]))
 
+;; Should this check directory name instead if file graphs also
+;; have this file?
 (defn db-graph-directory?
   "Returns boolean indicating if the given directory is a DB graph"
   [graph-dir]
-  (fs/existsSync (node-path/join graph-dir "db.sqlite")))
+  (fs/existsSync (node-path/join graph-dir "db.sqlite")))
+
+;; Reference same sqlite default class in cljs + nbb without needing .cljc
+(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
+
+(defn query
+  "Run a sql query against the given better-sqlite3 db"
+  [db sql]
+  (let [stmt (.prepare db sql)]
+    (.all ^object stmt)))
+
+(defn- upsert-addr-content!
+  "Upsert addr+data-seq"
+  [db data delete-addrs]
+  (let [insert (.prepare db "INSERT INTO kvs (addr, content) values (@addr, @content) on conflict(addr) do update set content = @content")
+        delete (.prepare db "DELETE from kvs where addr = ?")
+        insert-many (.transaction ^object db
+                                  (fn [data]
+                                    (doseq [item data]
+                                      (.run ^object insert item))
+                                    (doseq [addr delete-addrs]
+                                      (when addr
+                                        (.run ^object delete addr)))))]
+    (insert-many data)))
+
+(defn- restore-data-from-addr
+  [db addr]
+  (when-let [content (-> (query db (str "select content from kvs where addr = " addr))
+                         first
+                         (gobj/get "content"))]
+    (try
+      (let [data (sqlite-util/transit-read content)]
+        (if-let [addresses (:addresses data)]
+          (assoc data :addresses (clj->js addresses))
+          data))
+      (catch :default _e              ; TODO: remove this once db goes to test
+        (edn/read-string content)))))
+
+(defn new-sqlite-storage
+  "Creates a datascript storage for sqlite. Should be functionally equivalent to db-worker/new-sqlite-storage"
+  [db]
+  (reify IStorage
+    (-store [_ addr+data-seq delete-addrs]
+      (let [data (->>
+                  (map
+                   (fn [[addr data]]
+                     #js {:addr addr
+                          :content (sqlite-util/transit-write data)})
+                   addr+data-seq)
+                  (to-array))]
+        (upsert-addr-content! db data delete-addrs)))
+    (-restore [_ addr]
+      (restore-data-from-addr db addr))))
+
+(defn open-db!
+  "For a given database name, opens a sqlite db connection for it, creates
+  needed sqlite tables if not created and returns a datascript connection that's
+  connected to the sqlite db"
+  [graphs-dir db-name]
+  (let [[_db-sanitized-name db-full-path] (sqlite-common-db/get-db-full-path graphs-dir db-name)
+        db (new sqlite db-full-path nil)
+        ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
+        schema (if (sqlite-util/local-file-based-graph? db-name)
+                 db-schema/schema
+                 db-schema/schema-for-db-based-graph)]
+    (sqlite-common-db/create-kvs-table! db)
+    (let [storage (new-sqlite-storage db)
+          conn (sqlite-common-db/get-storage-conn storage schema)]
+      conn)))

deps/db/src/logseq/db/sqlite/db.cljs

@@ -1,107 +0,0 @@
-(ns ^:node-only logseq.db.sqlite.db
-  "Sqlite fns for db graphs"
-  (:require ["better-sqlite3" :as sqlite3]
-            [logseq.db.sqlite.common-db :as sqlite-common-db]
-            ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
-            #_:clj-kondo/ignore
-            [datascript.core :as d]
-            [datascript.storage :refer [IStorage]]
-            [goog.object :as gobj]
-            [clojure.edn :as edn]
-            [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.sqlite.util :as sqlite-util]))
-
-;; Reference same sqlite default class in cljs + nbb without needing .cljc
-(def sqlite (if (find-ns 'nbb.core) (aget sqlite3 "default") sqlite3))
-
-;; sqlite databases
-(defonce databases (atom nil))
-;; datascript conns
-(defonce conns (atom nil))
-
-(defn close!
-  []
-  (when @databases
-    (doseq [[_ database] @databases]
-      (.close database))
-    (reset! databases nil)))
-
-(def sanitize-db-name sqlite-common-db/sanitize-db-name)
-
-(def get-db-full-path sqlite-common-db/get-db-full-path)
-
-(defn get-conn
-  [repo]
-  (get @conns (sanitize-db-name repo)))
-
-(defn query
-  [db sql]
-  (let [stmt (.prepare db sql)]
-    (.all ^object stmt)))
-
-(defn upsert-addr-content!
-  "Upsert addr+data-seq"
-  [db data delete-addrs]
-  (let [insert (.prepare db "INSERT INTO kvs (addr, content) values (@addr, @content) on conflict(addr) do update set content = @content")
-        delete (.prepare db "DELETE from kvs where addr = ?")
-        insert-many (.transaction ^object db
-                                  (fn [data]
-                                    (doseq [item data]
-                                      (.run ^object insert item))
-                                    (doseq [addr delete-addrs]
-                                      (when addr
-                                        (.run ^object delete addr)))))]
-    (insert-many data)))
-
-(defn restore-data-from-addr
-  [db addr]
-  (when-let [content (-> (query db (str "select content from kvs where addr = " addr))
-                         first
-                         (gobj/get "content"))]
-    (try
-        (let [data (sqlite-util/transit-read content)]
-         (if-let [addresses (:addresses data)]
-           (assoc data :addresses (clj->js addresses))
-           data))
-        (catch :default _e              ; TODO: remove this once db goes to test
-          (edn/read-string content)))))
-
-(defn new-sqlite-storage
-  "Creates a datascript storage for sqlite. Should be functionally equivalent to db-worker/new-sqlite-storage"
-  [db]
-  (reify IStorage
-    (-store [_ addr+data-seq delete-addrs]
-      (let [data (->>
-                  (map
-                   (fn [[addr data]]
-                     #js {:addr addr
-                          :content (sqlite-util/transit-write data)})
-                   addr+data-seq)
-                  (to-array))]
-        (upsert-addr-content! db data delete-addrs)))
-    (-restore [_ addr]
-      (restore-data-from-addr db addr))))
-
-(defn open-db!
-  "For a given database name, opens a sqlite db connection for it, creates
-  needed sqlite tables if not created and returns a datascript connection that's
-  connected to the sqlite db"
-  [graphs-dir db-name]
-  (let [[db-sanitized-name db-full-path] (get-db-full-path graphs-dir db-name)
-        db (new sqlite db-full-path nil)
-        ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
-        schema (if (sqlite-util/local-file-based-graph? db-name)
-                 db-schema/schema
-                 db-schema/schema-for-db-based-graph)]
-    (sqlite-common-db/create-kvs-table! db)
-    (swap! databases assoc db-sanitized-name db)
-    (let [storage (new-sqlite-storage db)
-          conn (sqlite-common-db/get-storage-conn storage schema)]
-      (swap! conns assoc db-sanitized-name conn)
-      conn)))
-
-(defn transact!
-  [repo tx-data tx-meta]
-  (if-let [conn (get-conn repo)]
-    (d/transact! conn tx-data tx-meta)
-    (throw (ex-info (str "Failed to transact! No db connection found for " repo) {}))))

deps/db/test/logseq/db/sqlite/common_db_test.cljs

@@ -5,7 +5,7 @@
             [datascript.core :as d]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
             [logseq.common.util.date-time :as date-time-util]
-            [logseq.db.sqlite.db :as sqlite-db]
+            [logseq.db.sqlite.cli :as sqlite-cli]
             [clojure.string :as string]))
 
 (use-fixtures
@@ -27,7 +27,7 @@
   (testing "Fetches a defined block"
     (create-graph-dir "tmp/graphs" "test-db")
 
-    (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
+    (let [conn* (sqlite-cli/open-db! "tmp/graphs" "test-db")
           blocks [{:file/path "logseq/config.edn"
                    :file/content "{:foo :bar}"}]
           _ (d/transact! conn* blocks)
@@ -43,7 +43,7 @@
 (deftest restore-initial-data
   (testing "Restore a journal page"
     (create-graph-dir "tmp/graphs" "test-db")
-    (let [conn* (sqlite-db/open-db! "tmp/graphs" "test-db")
+    (let [conn* (sqlite-cli/open-db! "tmp/graphs" "test-db")
           page-uuid (random-uuid)
           block-uuid (random-uuid)
           created-at (js/Date.now)

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -4,7 +4,7 @@
             [logseq.graph-parser.test.docs-graph-helper :as docs-graph-helper]
             [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.db.sqlite.db :as sqlite-db]
+            [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.graph-parser.db :as gp-db]
             [clojure.set :as set]
             ["fs" :as fs]
@@ -129,7 +129,7 @@
   "Creates a sqlite-based db graph given a map of pages to blocks and returns a datascript db.
    Blocks in a map can only be top-level blocks with no referenced content"
   [dir db-name pages-to-blocks]
-  (let [conn (sqlite-db/open-db! dir db-name)
+  (let [conn (sqlite-cli/open-db! dir db-name)
         frontend-blocks (create-frontend-blocks pages-to-blocks)
         _ (d/transact! conn frontend-blocks)]
     (gp-db/create-default-pages! conn)

deps/outliner/script/transact.cljs

@@ -1,7 +1,7 @@
 (ns transact
   "This script generically runs transactions against the queried blocks"
   (:require [logseq.outliner.db-pipeline :as db-pipeline]
-            [logseq.db.sqlite.db :as sqlite-db]
+            [logseq.db.sqlite.cli :as sqlite-cli]
             [logseq.db.frontend.rules :as rules]
             [datascript.core :as d]
             [clojure.edn :as edn]
@@ -19,7 +19,7 @@
         [dir db-name] (if (string/includes? graph-dir "/")
                         ((juxt node-path/dirname node-path/basename) graph-dir)
                         [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        conn (sqlite-db/open-db! dir db-name)
+        conn (sqlite-cli/open-db! dir db-name)
         ;; find blocks to update
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries
         transact-fn (edn/read-string transact-fn*)

deps/outliner/src/logseq/outliner/cli.cljs

@@ -4,7 +4,7 @@
               [datascript.core :as d]
               [logseq.db.sqlite.create-graph :as sqlite-create-graph]
               [logseq.db.sqlite.build :as sqlite-build]
-              [logseq.db.sqlite.db :as sqlite-db]
+              [logseq.db.sqlite.cli :as sqlite-cli]
               [logseq.outliner.db-pipeline :as db-pipeline]
               ["fs" :as fs]
               ["path" :as node-path]))
@@ -38,7 +38,7 @@
   [dir db-name & [opts]]
   (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true})
   ;; Same order as frontend.db.conn/start!
-  (let [conn (sqlite-db/open-db! dir db-name)]
+  (let [conn (sqlite-cli/open-db! dir db-name)]
     (db-pipeline/add-listener conn)
     (setup-init-data conn opts)
     conn))

scripts/src/logseq/tasks/dev/publishing.cljs

@@ -2,7 +2,6 @@
   "Basic script for publishing from CLI"
   (:require [logseq.graph-parser.cli :as gp-cli]
             [logseq.publishing :as publishing]
-            [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.sqlite.cli :as sqlite-cli]
             ["fs" :as fs]
             ["path" :as node-path]
@@ -23,7 +22,7 @@
 
 (defn- publish-db-graph [static-dir graph-dir output-path opts]
   (let [db-name (node-path/basename graph-dir)
-        conn (sqlite-db/open-db! (node-path/dirname graph-dir) db-name)
+        conn (sqlite-cli/open-db! (node-path/dirname graph-dir) db-name)
         repo-config (-> (d/q '[:find ?content
                                :where [?b :file/path "logseq/config.edn"] [?b :file/content ?content]]
                              @conn)