fix: db and graph-parser nbb tests to pass locally

Updated sqlite fns to use updated approach. Deleted some unused
fns from old approach

deps/db/src/logseq/db/sqlite/cli.cljs

@@ -2,8 +2,6 @@
   "Primary ns to interact with DB graphs with node.js based CLIs"
   (:require [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.sqlite.restore :as sqlite-restore]
-            [cljs-bean.core :as bean]
-            [datascript.core :as d]
             ["fs" :as fs]
             ["path" :as node-path]))
 
@@ -16,10 +14,4 @@
   "Reads a given sqlite db and returns a datascript connection of its contents.
    The sqlite db is assumed to have already been opened by sqlite-db/open-db!"
   [db-name]
-  (let [{:keys [uuid->db-id-map conn]}
-        (sqlite-restore/restore-initial-data (bean/->js (sqlite-db/get-initial-data db-name)))
-        db (sqlite-restore/restore-other-data
-                conn
-                (sqlite-db/get-other-data db-name [])
-                uuid->db-id-map)]
-    (d/conn-from-db db)))
+  (sqlite-restore/restore-initial-data (sqlite-db/get-initial-data db-name)))

deps/db/src/logseq/db/sqlite/db.cljs

@@ -4,9 +4,11 @@
             ["better-sqlite3" :as sqlite3]
             [clojure.string :as string]
             [logseq.db.sqlite.util :as sqlite-util]
-            [datascript.storage :refer [IStorage]]
-            [cljs.cache :as cache]
+            ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
             [datascript.core :as d]
+            [datascript.storage :refer [IStorage]]
+            ;; Disable until used as it effects nbb
+            ;; [cljs.cache :as cache]
             [goog.object :as gobj]
             [logseq.db.frontend.schema :as db-schema]
             [clojure.edn :as edn]))
@@ -92,9 +94,8 @@
           (gobj/get "content")))))
 
 (defn sqlite-storage
-  [repo {:keys [threshold]
-         :or {threshold 4096}}]
-  (let [_cache (cache/lru-cache-factory {} :threshold threshold)]
+  [repo _ #_{:keys [threshold] :or {threshold 4096}}]
+  (let [_cache nil #_(cache/lru-cache-factory {} :threshold threshold)]
     (reify IStorage
       (-store [_ addr+data-seq]
         (let [data (->>

deps/db/src/logseq/db/sqlite/restore.cljs

@@ -1,123 +1,9 @@
 (ns logseq.db.sqlite.restore
-  "Fns to restore a sqlite database to a datascript one"
-  (:require [cognitect.transit :as transit]
-            [cljs-bean.core :as bean]
-            [clojure.string :as string]
-            [datascript.core :as d]
-            [goog.object :as gobj]
+  "Fns to restore data from a sqlite database to a datascript one"
+  (:require [datascript.core :as d]
             [logseq.db.frontend.schema :as db-schema]))
 
-(def ^:private t-reader (transit/reader :json))
-
-(defn- uuid-string?
-  [s]
-  (and (string? s)
-       (= (count s) 36)
-       (string/includes? s "-")))
-
-(defn- eav->datom
-  [uuid->db-id-map [e a v]]
-  (let [v' (cond
-             (and (= :block/uuid a) (string? v))
-             (uuid v)
-
-             (and (coll? v) (= :block/uuid (first v)) (string? (second v)))
-             (get uuid->db-id-map (second v) v)
-
-             :else
-             v)]
-    (d/datom e a v')))
-
-(defn restore-other-data
-  "Given an existing datascript connection and additional sqlite data, returns a
-  new datascript db with the two combined"
-  [conn data uuid->db-id-map & [{:keys [init-db-fn] :or {init-db-fn d/init-db}}]]
-  (let [datoms (transient (set (d/datoms @conn :eavt)))]
-    (doseq [block data]
-      (let [uuid (gobj/get block "uuid")
-            eid (get uuid->db-id-map uuid)
-            _ (when (nil? eid)
-                (prn "Error: block without eid ")
-                (js/console.dir block))
-            _ (assert eid (str "Can't find eid " eid ", block: " block))
-            avs (->> (gobj/get block "datoms")
-                     (transit/read t-reader))]
-        (doseq [[a v] avs]
-          (when (not (#{:block/uuid :page_uuid} a))
-            (let [datom (eav->datom uuid->db-id-map [eid a v])]
-              (conj! datoms datom))))))
-
-    (let [all-datoms (persistent! datoms)
-          new-db (init-db-fn all-datoms db-schema/schema-for-db-based-graph)]
-      new-db)))
-
-(defn- datoms-str->eav-vec
-  "Given a block's `datoms` transit string and an associated entity id, returns
-  a vector of eav triples"
-  [datoms-str eid]
-  (->> datoms-str
-       (transit/read t-reader)
-       ;; Remove :page_uuid as it's a transient attribute used during restore but not in the UI
-       (remove #(= :page_uuid (first %)))
-       (mapv (partial apply vector eid))))
-
-(defn- restore-initial-data*
-  "Builds up most datom vectors including all that are assigned new db ids"
-  [assign-id-to-uuid-fn all-pages all-blocks init-data]
-  (let [pages-eav-coll (doall (mapcat (fn [page]
-                                        (let [eid (assign-id-to-uuid-fn (:uuid page))]
-                                          (datoms-str->eav-vec (:datoms page) eid)))
-                                      all-pages))
-        all-blocks' (doall
-                     (keep (fn [b]
-                             (let [eid (assign-id-to-uuid-fn (:uuid b))]
-                               (when (and (uuid-string? (:uuid b))
-                                          (uuid-string? (:page_uuid b)))
-                                 [[eid :block/uuid (:uuid b)]
-                                  [eid :block/page [:block/uuid (:page_uuid b)]]])))
-                           all-blocks))
-        init-data' (doall
-                    (keep (fn [b]
-                            (let [eid (assign-id-to-uuid-fn (:uuid b))]
-                              (if (and (uuid-string? (:uuid b))
-                                       (= 5 (:type b)))
-                                [[eid :block/uuid (:uuid b)]
-                                 [eid :block/unknown? true]]
-                                (datoms-str->eav-vec (:datoms b) eid))))
-                          init-data))]
-    {:pages-eav-coll pages-eav-coll
-     :all-blocks' all-blocks'
-     :init-data' init-data'}))
-
 (defn restore-initial-data
-  "Given initial sqlite data, returns a datascript connection and other data
-  needed for subsequent restoration"
-  [data & [{:keys [conn-from-datoms-fn] :or {conn-from-datoms-fn d/conn-from-datoms}}]]
-  (let [{:keys [all-pages all-blocks journal-blocks init-data]} (bean/->clj data)
-        uuid->db-id-tmap (transient (hash-map))
-        *next-db-id (atom 100001)
-        assign-id-to-uuid-fn (fn [uuid-str]
-                               (or
-                                (get uuid->db-id-tmap uuid-str)
-                                (let [id @*next-db-id]
-                                  (conj! uuid->db-id-tmap [uuid-str id])
-                                  (swap! *next-db-id inc)
-                                  id)))
-        {:keys [pages-eav-coll all-blocks' init-data']}
-        (restore-initial-data* assign-id-to-uuid-fn all-pages all-blocks init-data)
-        uuid->db-id-map (persistent! uuid->db-id-tmap)
-        journal-blocks' (mapv
-                         (fn [b]
-                           (let [eid (get uuid->db-id-map (:uuid b))]
-                             (datoms-str->eav-vec (:datoms b) eid)))
-                         journal-blocks)
-        blocks-eav-colls (->> (concat all-blocks' journal-blocks' init-data')
-                              (apply concat))
-        all-eav-coll (doall (concat pages-eav-coll blocks-eav-colls))
-        datoms (map (partial eav->datom uuid->db-id-map)
-                    all-eav-coll)
-        db-conn (conn-from-datoms-fn datoms db-schema/schema-for-db-based-graph)]
-    {:conn db-conn
-     :uuid->db-id-map uuid->db-id-map
-     :journal-blocks journal-blocks
-     :datoms-count (count datoms)}))
+  "Given initial sqlite data, returns a datascript connection"
+  [datoms]
+  (d/conn-from-datoms datoms db-schema/schema-for-db-based-graph))

deps/db/src/logseq/db/sqlite/util.cljs

@@ -2,9 +2,7 @@
   "Utils fns for backend sqlite db"
   (:require [cljs-time.coerce :as tc]
             [cljs-time.core :as t]
-            [clojure.string :as string]
-            [cognitect.transit :as transit]
-            [logseq.db.frontend.schema :as db-schema]))
+            [clojure.string :as string]))
 
 (defn- type-of-block
   "
@@ -49,27 +47,6 @@
    :created_at (or (:block/created-at b) (time-ms))
    :updated_at (or (:block/updated-at b) (time-ms))})
 
-(defn block-map->datoms-str
-  "Given a block map and all existing blocks, return the block as transit data
-   to be stored in the `datoms` column. This is currently only used in testing"
-  [blocks m]
-  (let [t-writer (transit/writer :json)]
-    (->> (dissoc m :db/id)
-         ;; This fn should match pipeline/datom->av-vector
-         (map (fn m->av-vector [[a v]]
-                [a v]
-                (cond
-                  (contains? db-schema/card-one-ref-type-attributes a)
-                  [a [:block/uuid (str (some #(when (= (:db/id %) (:db/id v)) (:block/uuid %)) blocks))]]
-
-                  (contains? db-schema/card-many-ref-type-attributes a)
-                  [a (seq
-                      (map (fn [{db-id :db/id}]
-                             [:block/uuid (some #(when (= db-id (:db/id %)) (:block/uuid %)) blocks)])
-                           v))]
-                  :else [a v])))
-         (transit/write t-writer))))
-
 (defn block-with-timestamps
   "Adds updated-at timestamp and created-at if it doesn't exist"
   [block]

deps/db/test/logseq/db/sqlite/db_test.cljs

@@ -2,9 +2,9 @@
   (:require [cljs.test :refer [deftest async use-fixtures is testing]]
             ["fs" :as fs]
             ["path" :as node-path]
-            [cljs-bean.core :as bean]
+            [datascript.core :as d]
             [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.restore :as sqlite-restore]))
 
 (use-fixtures
   :each
@@ -22,120 +22,17 @@
   (fs/mkdirSync (node-path/join dir db-name) #js {:recursive true}))
 
 (deftest get-initial-data
-  (testing "Fetches file block"
+  (testing "Fetches a defined block"
     (create-graph-dir "tmp/graphs" "test-db")
     (sqlite-db/open-db! "tmp/graphs" "test-db")
-    (let [blocks (mapv sqlite-util/ds->sqlite-block
-                       [{:block/uuid (random-uuid)
-                         :file/path "logseq/config.edn"
-                         :file/content "{:foo :bar}"}])
-          _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))]
-      (is (= {:content "{:foo :bar}"
-              :name "logseq/config.edn"
-              :type 3}
-             (-> (sqlite-db/get-initial-data "test-db")
-                 :init-data
-                 bean/->clj
-                 first
-                 (select-keys [:content :name :type])))
-          "Correct file with content is found"))))
-
-(deftest upsert-blocks!
-  (let [page-uuid (random-uuid)
-        block-uuid (random-uuid)
-        created-at 1688054127299]
-    (create-graph-dir "tmp/graphs" "test-db")
-    (sqlite-db/open-db! "tmp/graphs" "test-db")
-
-    (testing "Creates a journal block"
-      (let [blocks (mapv sqlite-util/ds->sqlite-block
-                         [{:block/uuid page-uuid
-                           :block/journal-day 20230629
-                           :block/name "jun 29th, 2023"
-                           :block/created-at created-at
-                           :block/updated-at created-at}
-                          {:block/content "test"
-                           :block/uuid block-uuid
-                           :block/page {:db/id 100022}
-                           :block/created-at created-at
-                           :block/updated-at created-at
-                           :page_uuid page-uuid}])
-            _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
-            db-data (sqlite-db/get-initial-data "test-db")]
-        (is (= {:uuid (str page-uuid) :page_journal_day 20230629
-                :name "jun 29th, 2023" :type 2
-                :created_at created-at}
-               (-> db-data
-                   :all-pages
-                   first
-                   bean/->clj
-                   (select-keys [:uuid :page_journal_day :type :name :created_at])))
-            "New journal page is saved")
-
-        (is (= {:content "test" :name nil
-                :uuid (str block-uuid) :type 1
-                :created_at created-at}
-               (-> db-data
-                   :journal-blocks
-                   first
-                   bean/->clj
-                   (select-keys [:uuid :type :content :name :created_at])))
-            "New journal block content is saved")
-
-        (is (= [{:uuid (str block-uuid) :page_uuid (str page-uuid)}]
-               (-> db-data :all-blocks bean/->clj))
-            "Correct block and page uuid pairs exist")))
-
-    (testing "Updates a block"
-      (let [updated-at 1688072416134
-            blocks (mapv sqlite-util/ds->sqlite-block
-                         [{:block/uuid page-uuid
-                           :block/journal-day 20230629
-                           :block/name "jun 29th, 2023"
-                           :block/created-at created-at
-                           :block/updated-at updated-at}
-                          {:block/content "test edit"
-                           :block/uuid block-uuid
-                           :block/page {:db/id 100022}
-                           :block/created-at created-at
-                           :block/updated-at updated-at
-                           :page_uuid page-uuid}])
-            _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
-            db-data (sqlite-db/get-initial-data "test-db")]
-        (is (= {:uuid (str page-uuid) :updated_at updated-at :created_at created-at}
-               (-> db-data
-                   :all-pages
-                   first
-                   bean/->clj
-                   (select-keys [:uuid :updated_at :created_at])))
-            "Updated page has correct timestamps")
-
-        (is (= {:content "test edit" :created_at created-at :updated_at updated-at}
-               (-> db-data
-                   :journal-blocks
-                   first
-                   bean/->clj
-                   (select-keys [:content :created_at :updated_at])))
-            "Updated block has correct content and timestamps")))))
-
-(deftest get-other-data
-  (testing "Retrieves a normal page block"
-    (create-graph-dir "tmp/graphs" "test-db")
-    (sqlite-db/open-db! "tmp/graphs" "test-db")
-    (let [page-uuid (random-uuid)
-          block-uuid (random-uuid)
-          blocks (mapv sqlite-util/ds->sqlite-block
-                       [{:block/uuid page-uuid
-                         :block/name "some page"}
-                        {:block/content "test"
-                         :block/uuid block-uuid
-                         :block/page {:db/id 100022}
-                         :page_uuid page-uuid}])]
-      (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
-      (is (= {:content "test" :uuid (str block-uuid)
-              :page_uuid (str page-uuid) :type 1}
-             (-> (sqlite-db/get-other-data "test-db" [])
-                 bean/->clj
-                 first
-                 (select-keys [:content :page_uuid :type :uuid])))
-          "New page block is fetched with get-other-data"))))
+    (let [blocks [{:block/uuid (random-uuid)
+                   :file/path "logseq/config.edn"
+                   :file/content "{:foo :bar}"}]
+          _ (sqlite-db/transact! "test-db" blocks {})]
+      (is (= blocks
+             (->> (sqlite-db/get-initial-data "test-db")
+                  sqlite-restore/restore-initial-data
+                  deref
+                  (d/q '[:find (pull ?b [:block/uuid :file/path :file/content]) :where [?b :file/content]])
+                  (map first)))
+          "Correct file with content is found"))))

deps/db/test/logseq/db/sqlite/restore_test.cljs

@@ -2,11 +2,9 @@
   (:require [cljs.test :refer [deftest async use-fixtures is testing]]
             ["fs" :as fs]
             ["path" :as node-path]
-            [cljs-bean.core :as bean]
             [datascript.core :as d]
             [logseq.db.sqlite.db :as sqlite-db]
-            [logseq.db.sqlite.restore :as sqlite-restore]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.restore :as sqlite-restore]))
 
 (use-fixtures
   :each
@@ -41,50 +39,13 @@
                             :block/uuid block-uuid
                             :block/page {:db/id 100001}
                             :block/created-at created-at
-                            :block/updated-at created-at
-                            :page_uuid page-uuid}]
-          blocks (mapv #(sqlite-util/ds->sqlite-block
-                         (assoc % :datoms (sqlite-util/block-map->datoms-str frontend-blocks %)))
-                       frontend-blocks)
-          _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
-          {:keys [conn]} (sqlite-restore/restore-initial-data (bean/->js (sqlite-db/get-initial-data "test-db")))]
+                            :block/updated-at created-at}]
+          _ (sqlite-db/transact! "test-db" frontend-blocks {})
+          conn (-> (sqlite-db/get-initial-data "test-db")
+                   sqlite-restore/restore-initial-data)]
       (is (= (map #(dissoc % :page_uuid) frontend-blocks)
              (->> (d/q '[:find (pull ?b [*])
                          :where [?b :block/created-at]]
                        @conn)
                   (map first)))
-          "Datascript db matches data inserted into sqlite from simulated frontend"))))
-
-(deftest restore-other-data
-  (testing "Restore a page with its block"
-    (create-graph-dir "tmp/graphs" "test-db")
-    (sqlite-db/open-db! "tmp/graphs" "test-db")
-    (let [page-uuid (random-uuid)
-          block-uuid (random-uuid)
-          created-at (js/Date.now)
-          frontend-blocks [{:db/id 100001
-                            :block/uuid page-uuid
-                            :block/name "some page"
-                            :block/created-at created-at}
-                           {:db/id 100002
-                            :block/content "test"
-                            :block/uuid block-uuid
-                            :block/page {:db/id 100001}
-                            :page_uuid page-uuid
-                            :block/created-at created-at}]
-          blocks (mapv #(sqlite-util/ds->sqlite-block
-                         (assoc % :datoms (sqlite-util/block-map->datoms-str frontend-blocks %)))
-                       frontend-blocks)
-          _ (sqlite-db/upsert-blocks! "test-db" (bean/->js blocks))
-          {:keys [uuid->db-id-map conn]}
-          (sqlite-restore/restore-initial-data (bean/->js (sqlite-db/get-initial-data "test-db")))
-          new-db (sqlite-restore/restore-other-data
-                  conn
-                  (sqlite-db/get-other-data "test-db" [])
-                  uuid->db-id-map)]
-      (is (= (map #(dissoc % :page_uuid) frontend-blocks)
-             (->> (d/q '[:find (pull ?b [*])
-                         :where [?b :block/created-at]]
-                       new-db)
-                  (map first)))
           "Datascript db matches data inserted into sqlite from simulated frontend"))))

deps/graph-parser/test/logseq/graph_parser/cli_test.cljs

@@ -6,8 +6,6 @@
             [datascript.core :as d]
             [logseq.db.sqlite.db :as sqlite-db]
             [logseq.db.sqlite.cli :as sqlite-cli]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [cljs-bean.core :as bean]
             [logseq.db :as ldb]
             [clojure.set :as set]
             ["fs" :as fs]
@@ -115,7 +113,6 @@
                 (mapv #(merge %
                               {:db/id (new-db-id)
                                :block/uuid (random-uuid)
-                               :page_uuid page-uuid
                                :block/format :markdown
                                :block/path-refs [{:db/id page-id}]
                                :block/page {:db/id page-id}
@@ -132,10 +129,7 @@
   [dir db-name pages-to-blocks]
   (sqlite-db/open-db! dir db-name)
   (let [frontend-blocks (create-frontend-blocks pages-to-blocks)
-        blocks (mapv #(sqlite-util/ds->sqlite-block
-                       (assoc % :datoms (sqlite-util/block-map->datoms-str frontend-blocks %)))
-                     frontend-blocks)
-        _ (sqlite-db/upsert-blocks! db-name (bean/->js blocks))
+        _ (sqlite-db/transact! db-name frontend-blocks {})
         conn (sqlite-cli/read-graph db-name)]
     (ldb/create-default-pages! conn {:db-graph? true})
     @conn))