refactor: split logseq.db.common.sqlite into 2 namespaces

The majority of this ns is focused on graph initilization and
entity lazy loading so split this out to new ns,
logseq.db.common.initial-data. Originally
initialization was tied to sqlite util fns but it has not made
sense for awhile to think of them as sqlite util fns

.clj-kondo/config.edn

@@ -173,10 +173,12 @@
              logseq.common.util.namespace ns-util
              logseq.common.util.page-ref page-ref
              logseq.db ldb
+             logseq.db.common.entity-plus entity-plus
              logseq.db.common.entity-util common-entity-util
+             logseq.db.common.initial-data common-initial-data
              logseq.db.common.order db-order
              logseq.db.common.property-util db-property-util
-             logseq.db.common.sqlite sqlite-common-db
+             logseq.db.common.sqlite common-sqlite
              logseq.db.common.view db-view
              logseq.db.file-based.rules file-rules
              logseq.db.file-based.schema file-schema
@@ -185,7 +187,6 @@
              logseq.db.frontend.content db-content
              logseq.db.frontend.db db-db
              logseq.db.frontend.db-ident db-ident
-             logseq.db.common.entity-plus entity-plus
              logseq.db.frontend.entity-util entity-util
              logseq.db.frontend.inputs db-inputs
              logseq.db.frontend.property db-property

deps/db/.carve/ignore

@@ -28,3 +28,9 @@ logseq.db.sqlite.export/build-import
 logseq.db.common.view/get-property-values
 ;; API
 logseq.db.common.view/get-view-data
+;; API
+logseq.db.common.initial-data/with-parent
+;; API
+logseq.db.common.initial-data/get-block-and-children
+;; API
+logseq.db.common.initial-data/get-initial-data

deps/db/.clj-kondo/config.edn

@@ -9,10 +9,11 @@
   :consistent-alias
   {:aliases {clojure.string string
              logseq.db ldb
+             logseq.db.common.entity-plus entity-plus
              logseq.db.common.entity-util common-entity-util
              logseq.db.common.order db-order
              logseq.db.common.property-util db-property-util
-             logseq.db.common.sqlite sqlite-common-db
+             logseq.db.common.sqlite common-sqlite
              logseq.db.common.view db-view
              logseq.db.frontend.content db-content
              logseq.db.frontend.class db-class
@@ -25,11 +26,11 @@
              logseq.db.file-based.rules file-rules
              logseq.db.file-based.schema file-schema
              logseq.db.file-based.entity-util file-entity-util
-             logseq.db.common.entity-plus entity-plus
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate
              logseq.db.sqlite.build sqlite-build
+             logseq.db.common.initial-data common-initial-data
              logseq.db.common.sqlite-cli sqlite-cli
              logseq.db.sqlite.create-graph sqlite-create-graph
              logseq.db.sqlite.export sqlite-export

deps/db/src/logseq/db.cljs

@@ -12,7 +12,7 @@
             [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
             [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.entity-util :as common-entity-util]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.db :as db-db]
@@ -192,7 +192,7 @@
        (map first)
        (remove hidden?)))
 
-(def get-first-page-by-name sqlite-common-db/get-first-page-by-name)
+(def get-first-page-by-name common-initial-data/get-first-page-by-name)
 
 (def db-based-graph? entity-plus/db-based-graph?)
 
@@ -250,7 +250,7 @@
     (if-let [id (if (uuid? page-name-or-uuid) page-name-or-uuid
                     (parse-uuid page-name-or-uuid))]
       (d/entity db [:block/uuid id])
-      (d/entity db (sqlite-common-db/get-first-page-by-title db page-name-or-uuid)))))
+      (d/entity db (common-initial-data/get-first-page-by-title db page-name-or-uuid)))))
 
 (defn page-empty?
   "Whether a page is empty. Does it has a non-page block?
@@ -349,8 +349,8 @@
         (recur (:block/uuid parent) (conj parents' parent) (inc d))
         parents'))))
 
-(def get-block-children-ids sqlite-common-db/get-block-children-ids)
-(def get-block-children sqlite-common-db/get-block-children)
+(def get-block-children-ids common-initial-data/get-block-children-ids)
+(def get-block-children common-initial-data/get-block-children)
 
 (defn- get-sorted-page-block-ids
   [db page-id]
@@ -543,9 +543,9 @@
 (def get-class-ident-by-display-type db-db/get-class-ident-by-display-type)
 (def get-display-type-by-class-ident db-db/get-display-type-by-class-ident)
 
-(def get-recent-updated-pages sqlite-common-db/get-recent-updated-pages)
+(def get-recent-updated-pages common-initial-data/get-recent-updated-pages)
 
-(def get-latest-journals sqlite-common-db/get-latest-journals)
+(def get-latest-journals common-initial-data/get-latest-journals)
 
 (defn get-pages-relation
   [db with-journal?]

deps/db/src/logseq/db/common/initial_data.cljs

@@ -0,0 +1,321 @@
+(ns logseq.db.common.initial-data
+  "Provides db helper fns for graph initialization and lazy loading entities"
+  (:require [clojure.set :as set]
+            [datascript.core :as d]
+            [datascript.impl.entity :as de]
+            [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
+            [logseq.common.util.date-time :as date-time-util]
+            [logseq.db.common.entity-util :as common-entity-util]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.frontend.entity-util :as entity-util]))
+
+(defn- get-pages-by-name
+  [db page-name]
+  (d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
+
+(defn get-first-page-by-name
+  "Return the oldest page's db id for :block/name"
+  [db page-name]
+  (when (and db (string? page-name))
+    (first (sort (map :e (get-pages-by-name db page-name))))))
+
+(defn get-first-page-by-title
+  "Return the oldest page's db id for :block/title"
+  [db page-name]
+  {:pre [(string? page-name)]}
+  (->> (d/datoms db :avet :block/title page-name)
+       (filter (fn [d]
+                 (let [e (d/entity db (:e d))]
+                   (common-entity-util/page? e))))
+       (map :e)
+       sort
+       first))
+
+(comment
+  (defn- get-built-in-files
+    [db]
+    (let [files ["logseq/config.edn"
+                 "logseq/custom.css"
+                 "logseq/custom.js"]]
+      (map #(d/pull db '[*] [:file/path %]) files))))
+
+(defn- get-all-files
+  [db]
+  (->> (d/datoms db :avet :file/path)
+       (mapcat (fn [e] (d/datoms db :eavt (:e e))))))
+
+(defn- with-block-refs
+  [db block]
+  (update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
+
+(defn with-parent
+  [db block]
+  (cond
+    (:block/page block)
+    (let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
+                   (select-keys e [:db/id :block/uuid]))]
+      (->>
+       (assoc block :block/parent parent)
+       (common-util/remove-nils-non-nested)
+       (with-block-refs db)))
+
+    :else
+    block))
+
+(comment
+  (defn- with-block-link
+    [db block]
+    (if (:block/link block)
+      (update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
+      block)))
+
+(defn- mark-block-fully-loaded
+  [b]
+  (assoc b :block.temp/fully-loaded? true))
+
+(comment
+  (defn- property-without-db-attrs
+    [property]
+    (dissoc property :db/index :db/valueType :db/cardinality))
+
+  (defn- property-with-values
+    [db block properties]
+    (when (entity-plus/db-based-graph? db)
+      (let [block (d/entity db (:db/id block))
+            property-vals (if properties
+                            (map block properties)
+                            (vals (:block/properties block)))]
+        (->> property-vals
+             (mapcat
+              (fn [property-values]
+                (let [values (->>
+                              (if (and (coll? property-values)
+                                       (map? (first property-values)))
+                                property-values
+                                #{property-values}))
+                      value-ids (when (every? map? values)
+                                  (->> (map :db/id values)
+                                       (filter (fn [id] (or (int? id) (keyword? id))))))
+                      value-blocks (->>
+                                    (when (seq value-ids)
+                                      (map
+                                       (fn [id] (d/pull db '[:db/id :block/uuid
+                                                             :block/name :block/title
+                                                             :logseq.property/value
+                                                             :block/tags :block/page
+                                                             :logseq.property/created-from-property] id))
+                                       value-ids))
+                                  ;; FIXME: why d/pull returns {:db/id db-ident} instead of {:db/id number-eid}?
+                                    (keep (fn [block]
+                                            (let [from-property-id (get-in block [:logseq.property/created-from-property :db/id])]
+                                              (if (keyword? from-property-id)
+                                                (assoc-in block [:logseq.property/created-from-property :db/id] (:db/id (d/entity db from-property-id)))
+                                                block)))))]
+                  value-blocks))))))))
+
+(defn get-block-children-ids
+  "Returns children UUIDs"
+  [db block-uuid]
+  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
+    (let [seen   (volatile! [])]
+      (loop [steps          100      ;check result every 100 steps
+             eids-to-expand [eid]]
+        (when (seq eids-to-expand)
+          (let [eids-to-expand*
+                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
+                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
+            (when (and (zero? steps)
+                       (seq (set/intersection (set @seen) (set uuids-to-add))))
+              (throw (ex-info "bad outliner data, need to re-index to fix"
+                              {:seen @seen :eids-to-expand eids-to-expand})))
+            (vswap! seen (partial apply conj) uuids-to-add)
+            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
+      @seen)))
+
+(defn get-block-children
+  "Including nested children."
+  [db block-uuid]
+  (let [ids (get-block-children-ids db block-uuid)]
+    (when (seq ids)
+      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
+        (d/pull-many db '[*] ids')))))
+
+(defn- with-raw-title
+  [m entity]
+  (if-let [raw-title (:block/raw-title entity)]
+    (assoc m :block/title raw-title)
+    m))
+
+(defn- entity->map
+  [entity]
+  (-> (into {} entity)
+      (with-raw-title entity)
+      (assoc :db/id (:db/id entity))))
+
+(defn ^:large-vars/cleanup-todo get-block-and-children
+  [db id {:keys [children? children-only? nested-children? properties children-props]}]
+  (let [block (d/entity db (if (uuid? id)
+                             [:block/uuid id]
+                             id))
+        block-refs-count? (some #{:block.temp/refs-count} properties)
+        whiteboard? (common-entity-util/whiteboard? block)]
+    (when block
+      (let [children (when (or children? children-only?)
+                       (let [page? (common-entity-util/page? block)
+                             children (->>
+                                       (cond
+                                         (and nested-children? (not page?))
+                                         (get-block-children db (:block/uuid block))
+                                         nested-children?
+                                         (:block/_page block)
+                                         :else
+                                         (let [short-page? (when page?
+                                                             (<= (count (:block/_page block)) 100))]
+                                           (if short-page?
+                                             (:block/_page block)
+                                             (:block/_parent block))))
+                                       (remove (fn [e] (or (:logseq.property/created-from-property e)
+                                                           (:block/closed-value-property e)))))
+                             children-props (if whiteboard?
+                                              '[*]
+                                              (or children-props
+                                                  [:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
+                                                   ;; pre-loading feature-related properties to avoid UI refreshing
+                                                   :logseq.task/status :logseq.property.node/display-type]))]
+                         (map
+                          (fn [block]
+                            (if (= children-props '[*])
+                              (entity->map block)
+                              (-> (select-keys block children-props)
+                                  (with-raw-title block))))
+                          children)))]
+        (if children-only?
+          {:children children}
+          (let [block' (if (seq properties)
+                         (-> (select-keys block properties)
+                             (with-raw-title block)
+                             (assoc :db/id (:db/id block)))
+                         (entity->map block))
+                block' (cond->
+                        (mark-block-fully-loaded block')
+                         true
+                         (update-vals (fn [v]
+                                        (cond
+                                          (de/entity? v)
+                                          (entity->map v)
+                                          (and (coll? v) (every? de/entity? v))
+                                          (map entity->map v)
+
+                                          :else
+                                          v)))
+                         block-refs-count?
+                         (assoc :block.temp/refs-count (count (:block/_refs block))))]
+            (cond->
+             {:block block'}
+              children?
+              (assoc :children children))))))))
+
+(defn get-latest-journals
+  [db]
+  (let [today (date-time-util/date->int (js/Date.))]
+    (->> (d/datoms db :avet :block/journal-day)
+         vec
+         rseq
+         (keep (fn [d]
+                 (and (<= (:v d) today)
+                      (let [e (d/entity db (:e d))]
+                        (when (and (common-entity-util/journal? e) (:db/id e))
+                          e))))))))
+
+(defn- get-structured-datoms
+  [db]
+  (let [class-property-id (:db/id (d/entity db :logseq.class/Property))]
+    (->> (concat
+          (d/datoms db :avet :block/tags :logseq.class/Tag)
+          (d/datoms db :avet :block/tags :logseq.class/Property)
+          (d/datoms db :avet :block/closed-value-property))
+         (mapcat (fn [d]
+                   (let [block-datoms (d/datoms db :eavt (:e d))
+                         property-desc-datoms (when (= (:v d) class-property-id)
+                                                (when-let [desc (:logseq.property/description (d/entity db (:e d)))]
+                                                  (d/datoms db :eavt (:db/id desc))))]
+                     (if property-desc-datoms
+                       (concat block-datoms property-desc-datoms)
+                       block-datoms)))))))
+
+(defn- get-favorites
+  "Favorites page and its blocks"
+  [db]
+  (let [page-id (get-first-page-by-name db common-config/favorites-page-name)
+        block (d/entity db page-id)
+        children (:block/_page block)]
+    (when block
+      (concat (d/datoms db :eavt (:db/id block))
+              (->> (keep :block/link children)
+                   (mapcat (fn [l]
+                             (d/datoms db :eavt (:db/id l)))))
+              (mapcat (fn [child]
+                        (d/datoms db :eavt (:db/id child)))
+                      children)))))
+
+(defn- get-views-data
+  [db]
+  (let [page-id (get-first-page-by-name db common-config/views-page-name)
+        children (when page-id (:block/_parent (d/entity db page-id)))]
+    (when (seq children)
+      (into
+       (mapcat (fn [b] (d/datoms db :eavt (:db/id b)))
+               children)
+       (d/datoms db :eavt page-id)))))
+
+(defn get-recent-updated-pages
+  [db]
+  (some->>
+   (d/datoms db :avet :block/updated-at)
+   rseq
+   (keep (fn [datom]
+           (let [e (d/entity db (:e datom))]
+             (when (and (common-entity-util/page? e) (not (entity-util/hidden? e)))
+               e))))
+   (take 30)))
+
+(defn get-initial-data
+  "Returns current database schema and initial data.
+   NOTE: This fn is called by DB and file graphs"
+  [db]
+  (let [db-graph? (entity-plus/db-based-graph? db)
+        _ (when db-graph?
+            (reset! db-order/*max-key (db-order/get-max-order db)))
+        schema (:schema db)
+        idents (mapcat (fn [id]
+                         (when-let [e (d/entity db id)]
+                           (d/datoms db :eavt (:db/id e))))
+                       [:logseq.kv/db-type
+                        :logseq.kv/schema-version
+                        :logseq.kv/graph-uuid
+                        :logseq.kv/latest-code-lang
+                        :logseq.kv/graph-backup-folder
+                        :logseq.property/empty-placeholder])
+        favorites (when db-graph? (get-favorites db))
+        views (when db-graph? (get-views-data db))
+        all-files (get-all-files db)
+        structured-datoms (when db-graph?
+                            (get-structured-datoms db))
+        recent-updated-pages (let [pages (get-recent-updated-pages db)]
+                               (mapcat (fn [p] (d/datoms db :eavt (:db/id p))) pages))
+        pages-datoms (let [contents-id (get-first-page-by-title db "Contents")
+                           views-id (get-first-page-by-title db common-config/views-page-name)]
+                       (mapcat #(d/datoms db :eavt %)
+                               (remove nil? [contents-id views-id])))
+        data (distinct
+              (concat idents
+                      structured-datoms
+                      favorites
+                      recent-updated-pages
+                      views
+                      all-files
+                      pages-datoms))]
+    {:schema schema
+     :initial-data data}))

deps/db/src/logseq/db/common/sqlite.cljs

@@ -1,343 +1,11 @@
 (ns logseq.db.common.sqlite
-  "Provides common sqlite db fns for file and DB graphs. These fns work on
+  "Provides common sqlite util fns for file and DB graphs. These fns work on
   browser and node"
   (:require ["path" :as node-path]
-            [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
-            [datascript.impl.entity :as de]
-            [logseq.common.config :as common-config]
-            [logseq.common.util :as common-util]
-            [logseq.common.util.date-time :as date-time-util]
-            [logseq.db.common.entity-util :as common-entity-util]
-            [logseq.db.common.order :as db-order]
-            [logseq.db.common.entity-plus :as entity-plus]
-            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.sqlite.util :as sqlite-util]))
 
-(defn- get-pages-by-name
-  [db page-name]
-  (d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
-
-(defn get-first-page-by-name
-  "Return the oldest page's db id for :block/name"
-  [db page-name]
-  (when (and db (string? page-name))
-    (first (sort (map :e (get-pages-by-name db page-name))))))
-
-(defn get-first-page-by-title
-  "Return the oldest page's db id for :block/title"
-  [db page-name]
-  {:pre [(string? page-name)]}
-  (->> (d/datoms db :avet :block/title page-name)
-       (filter (fn [d]
-                 (let [e (d/entity db (:e d))]
-                   (common-entity-util/page? e))))
-       (map :e)
-       sort
-       first))
-
-(comment
-  (defn- get-built-in-files
-    [db]
-    (let [files ["logseq/config.edn"
-                 "logseq/custom.css"
-                 "logseq/custom.js"]]
-      (map #(d/pull db '[*] [:file/path %]) files))))
-
-(defn get-all-files
-  [db]
-  (->> (d/datoms db :avet :file/path)
-       (mapcat (fn [e] (d/datoms db :eavt (:e e))))))
-
-(defn- with-block-refs
-  [db block]
-  (update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
-
-(defn with-parent
-  [db block]
-  (cond
-    (:block/page block)
-    (let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
-                   (select-keys e [:db/id :block/uuid]))]
-      (->>
-       (assoc block :block/parent parent)
-       (common-util/remove-nils-non-nested)
-       (with-block-refs db)))
-
-    :else
-    block))
-
-(comment
-  (defn- with-block-link
-    [db block]
-    (if (:block/link block)
-      (update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
-      block)))
-
-(defn- mark-block-fully-loaded
-  [b]
-  (assoc b :block.temp/fully-loaded? true))
-
-(comment
-  (defn- property-without-db-attrs
-    [property]
-    (dissoc property :db/index :db/valueType :db/cardinality))
-
-  (defn- property-with-values
-    [db block properties]
-    (when (entity-plus/db-based-graph? db)
-      (let [block (d/entity db (:db/id block))
-            property-vals (if properties
-                            (map block properties)
-                            (vals (:block/properties block)))]
-        (->> property-vals
-             (mapcat
-              (fn [property-values]
-                (let [values (->>
-                              (if (and (coll? property-values)
-                                       (map? (first property-values)))
-                                property-values
-                                #{property-values}))
-                      value-ids (when (every? map? values)
-                                  (->> (map :db/id values)
-                                       (filter (fn [id] (or (int? id) (keyword? id))))))
-                      value-blocks (->>
-                                    (when (seq value-ids)
-                                      (map
-                                       (fn [id] (d/pull db '[:db/id :block/uuid
-                                                             :block/name :block/title
-                                                             :logseq.property/value
-                                                             :block/tags :block/page
-                                                             :logseq.property/created-from-property] id))
-                                       value-ids))
-                                  ;; FIXME: why d/pull returns {:db/id db-ident} instead of {:db/id number-eid}?
-                                    (keep (fn [block]
-                                            (let [from-property-id (get-in block [:logseq.property/created-from-property :db/id])]
-                                              (if (keyword? from-property-id)
-                                                (assoc-in block [:logseq.property/created-from-property :db/id] (:db/id (d/entity db from-property-id)))
-                                                block)))))]
-                  value-blocks))))))))
-
-(defn get-block-children-ids
-  "Returns children UUIDs"
-  [db block-uuid]
-  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
-    (let [seen   (volatile! [])]
-      (loop [steps          100      ;check result every 100 steps
-             eids-to-expand [eid]]
-        (when (seq eids-to-expand)
-          (let [eids-to-expand*
-                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
-                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
-            (when (and (zero? steps)
-                       (seq (set/intersection (set @seen) (set uuids-to-add))))
-              (throw (ex-info "bad outliner data, need to re-index to fix"
-                              {:seen @seen :eids-to-expand eids-to-expand})))
-            (vswap! seen (partial apply conj) uuids-to-add)
-            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
-      @seen)))
-
-(defn get-block-children
-  "Including nested children."
-  [db block-uuid]
-  (let [ids (get-block-children-ids db block-uuid)]
-    (when (seq ids)
-      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
-        (d/pull-many db '[*] ids')))))
-
-(defn- with-raw-title
-  [m entity]
-  (if-let [raw-title (:block/raw-title entity)]
-    (assoc m :block/title raw-title)
-    m))
-
-(defn- entity->map
-  [entity]
-  (-> (into {} entity)
-      (with-raw-title entity)
-      (assoc :db/id (:db/id entity))))
-
-(defn ^:large-vars/cleanup-todo get-block-and-children
-  [db id {:keys [children? children-only? nested-children? properties children-props]}]
-  (let [block (d/entity db (if (uuid? id)
-                             [:block/uuid id]
-                             id))
-        block-refs-count? (some #{:block.temp/refs-count} properties)
-        whiteboard? (common-entity-util/whiteboard? block)]
-    (when block
-      (let [children (when (or children? children-only?)
-                       (let [page? (common-entity-util/page? block)
-                             children (->>
-                                       (cond
-                                         (and nested-children? (not page?))
-                                         (get-block-children db (:block/uuid block))
-                                         nested-children?
-                                         (:block/_page block)
-                                         :else
-                                         (let [short-page? (when page?
-                                                             (<= (count (:block/_page block)) 100))]
-                                           (if short-page?
-                                             (:block/_page block)
-                                             (:block/_parent block))))
-                                       (remove (fn [e] (or (:logseq.property/created-from-property e)
-                                                           (:block/closed-value-property e)))))
-                             children-props (if whiteboard?
-                                              '[*]
-                                              (or children-props
-                                                  [:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
-                                                   ;; pre-loading feature-related properties to avoid UI refreshing
-                                                   :logseq.task/status :logseq.property.node/display-type]))]
-                         (map
-                          (fn [block]
-                            (if (= children-props '[*])
-                              (entity->map block)
-                              (-> (select-keys block children-props)
-                                  (with-raw-title block))))
-                          children)))]
-        (if children-only?
-          {:children children}
-          (let [block' (if (seq properties)
-                         (-> (select-keys block properties)
-                             (with-raw-title block)
-                             (assoc :db/id (:db/id block)))
-                         (entity->map block))
-                block' (cond->
-                        (mark-block-fully-loaded block')
-                         true
-                         (update-vals (fn [v]
-                                        (cond
-                                          (de/entity? v)
-                                          (entity->map v)
-                                          (and (coll? v) (every? de/entity? v))
-                                          (map entity->map v)
-
-                                          :else
-                                          v)))
-                         block-refs-count?
-                         (assoc :block.temp/refs-count (count (:block/_refs block))))]
-            (cond->
-             {:block block'}
-              children?
-              (assoc :children children))))))))
-
-(defn get-latest-journals
-  [db]
-  (let [today (date-time-util/date->int (js/Date.))]
-    (->> (d/datoms db :avet :block/journal-day)
-         vec
-         rseq
-         (keep (fn [d]
-                 (and (<= (:v d) today)
-                      (let [e (d/entity db (:e d))]
-                        (when (and (common-entity-util/journal? e) (:db/id e))
-                          e))))))))
-
-(defn get-page->refs-count
-  [db]
-  (let [datoms (d/datoms db :avet :block/name)]
-    (->>
-     (map (fn [d]
-            [(:e d)
-             (count (:block/_refs (d/entity db (:e d))))]) datoms)
-     (into {}))))
-
-(defn get-structured-datoms
-  [db]
-  (let [class-property-id (:db/id (d/entity db :logseq.class/Property))]
-    (->> (concat
-          (d/datoms db :avet :block/tags :logseq.class/Tag)
-          (d/datoms db :avet :block/tags :logseq.class/Property)
-          (d/datoms db :avet :block/closed-value-property))
-         (mapcat (fn [d]
-                   (let [block-datoms (d/datoms db :eavt (:e d))
-                         property-desc-datoms (when (= (:v d) class-property-id)
-                                                (when-let [desc (:logseq.property/description (d/entity db (:e d)))]
-                                                  (d/datoms db :eavt (:db/id desc))))]
-                     (if property-desc-datoms
-                       (concat block-datoms property-desc-datoms)
-                       block-datoms)))))))
-
-(defn get-favorites
-  "Favorites page and its blocks"
-  [db]
-  (let [page-id (get-first-page-by-name db common-config/favorites-page-name)
-        block (d/entity db page-id)
-        children (:block/_page block)]
-    (when block
-      (concat (d/datoms db :eavt (:db/id block))
-              (->> (keep :block/link children)
-                   (mapcat (fn [l]
-                             (d/datoms db :eavt (:db/id l)))))
-              (mapcat (fn [child]
-                        (d/datoms db :eavt (:db/id child)))
-                      children)))))
-
-(defn get-views-data
-  [db]
-  (let [page-id (get-first-page-by-name db common-config/views-page-name)
-        children (when page-id (:block/_parent (d/entity db page-id)))]
-    (when (seq children)
-      (into
-       (mapcat (fn [b] (d/datoms db :eavt (:db/id b)))
-               children)
-       (d/datoms db :eavt page-id)))))
-
-(defn get-recent-updated-pages
-  [db]
-  (some->>
-   (d/datoms db :avet :block/updated-at)
-   rseq
-   (keep (fn [datom]
-           (let [e (d/entity db (:e datom))]
-             (when (and (common-entity-util/page? e) (not (entity-util/hidden? e)))
-               e))))
-   (take 30)))
-
-(defn get-initial-data
-  "Returns current database schema and initial data.
-   NOTE: This fn is called by DB and file graphs"
-  [db]
-  (let [db-graph? (entity-plus/db-based-graph? db)
-        _ (when db-graph?
-            (reset! db-order/*max-key (db-order/get-max-order db)))
-        schema (:schema db)
-        idents (mapcat (fn [id]
-                         (when-let [e (d/entity db id)]
-                           (d/datoms db :eavt (:db/id e))))
-                       [:logseq.kv/db-type
-                        :logseq.kv/schema-version
-                        :logseq.kv/graph-uuid
-                        :logseq.kv/latest-code-lang
-                        :logseq.kv/graph-backup-folder
-                        :logseq.property/empty-placeholder])
-        favorites (when db-graph? (get-favorites db))
-        views (when db-graph? (get-views-data db))
-        all-files (get-all-files db)
-        structured-datoms (when db-graph?
-                            (get-structured-datoms db))
-        recent-updated-pages (let [pages (get-recent-updated-pages db)]
-                               (mapcat (fn [p] (d/datoms db :eavt (:db/id p))) pages))
-        pages-datoms (let [contents-id (get-first-page-by-title db "Contents")
-                           views-id (get-first-page-by-title db common-config/views-page-name)]
-                       (mapcat #(d/datoms db :eavt %)
-                               (remove nil? [contents-id views-id])))
-        data (distinct
-              (concat idents
-                      structured-datoms
-                      favorites
-                      recent-updated-pages
-                      views
-                      all-files
-                      pages-datoms))]
-    {:schema schema
-     :initial-data data}))
-
-(defn restore-initial-data
-  "Given initial Datascript datoms and schema, returns a datascript connection"
-  [data schema]
-  (d/conn-from-datoms data schema))
-
 (defn create-kvs-table!
   "Creates a sqlite table for use with datascript.storage if one doesn't exist"
   [sqlite-db]

deps/db/src/logseq/db/common/sqlite_cli.cljs

@@ -7,7 +7,7 @@
             ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
             [datascript.core]
             [datascript.storage :refer [IStorage]]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.util :as sqlite-util]))
@@ -87,13 +87,13 @@
   needed sqlite tables if not created and returns a datascript connection that's
   connected to the sqlite db"
   [graphs-dir db-name]
-  (let [[_db-sanitized-name db-full-path] (sqlite-common-db/get-db-full-path graphs-dir db-name)
+  (let [[_db-sanitized-name db-full-path] (common-sqlite/get-db-full-path graphs-dir db-name)
         db (new sqlite db-full-path nil)
         ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
-        schema (if (sqlite-common-db/local-file-based-graph? db-name)
+        schema (if (common-sqlite/local-file-based-graph? db-name)
                  file-schema/schema
                  db-schema/schema)]
-    (sqlite-common-db/create-kvs-table! db)
+    (common-sqlite/create-kvs-table! db)
     (let [storage (new-sqlite-storage db)
-          conn (sqlite-common-db/get-storage-conn storage schema)]
+          conn (common-sqlite/get-storage-conn storage schema)]
       conn)))

deps/db/test/logseq/db/common/sqlite_test.cljs → deps/db/test/logseq/db/common/initial_data_test.cljs

@@ -1,11 +1,11 @@
-(ns logseq.db.common.sqlite-test
+(ns logseq.db.common.initial-data-test
   "This ns is the only one to test against file based datascript connections.
    These are useful integration tests"
   (:require ["fs" :as fs]
             ["path" :as node-path]
             [cljs.test :refer [deftest async use-fixtures is testing]]
             [datascript.core :as d]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.sqlite.build :as sqlite-build]
             [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
@@ -35,8 +35,8 @@
                    :file/content "{:foo :bar}"}]
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
-          {:keys [schema initial-data]} (sqlite-common-db/get-initial-data @conn*)
-          conn (sqlite-common-db/restore-initial-data initial-data schema)]
+          {:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
+          conn (d/conn-from-datoms initial-data schema)]
       (is (= blocks
              (->> @conn
                   (d/q '[:find (pull ?b [:block/uuid :file/path :file/content]) :where [?b :file/content]])
@@ -54,7 +54,7 @@
             :blocks [{:block/title "b1"}]}]})
         _ (d/transact! conn* init-tx)
           ;; Simulate getting data from sqlite and restoring it for frontend
-        {:keys [schema initial-data]} (sqlite-common-db/get-initial-data @conn*)
-        conn (sqlite-common-db/restore-initial-data initial-data schema)]
+        {:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
+        conn (d/conn-from-datoms initial-data schema)]
     (is (some? (db-test/find-page-by-title @conn "page1"))
         "Restores recently updated page")))

src/electron/electron/db.cljs

@@ -3,7 +3,7 @@
   (:require ["electron" :refer [app]]
             ["fs-extra" :as fs]
             ["path" :as node-path]
-            [logseq.db.common.sqlite :as sqlite-common-db]))
+            [logseq.db.common.sqlite :as common-sqlite]))
 
 (defn get-graphs-dir
   []
@@ -17,19 +17,19 @@
 (defn ensure-graph-dir!
   [db-name]
   (ensure-graphs-dir!)
-  (let [graph-dir (node-path/join (get-graphs-dir) (sqlite-common-db/sanitize-db-name db-name))]
+  (let [graph-dir (node-path/join (get-graphs-dir) (common-sqlite/sanitize-db-name db-name))]
     (fs/ensureDirSync graph-dir)
     graph-dir))
 
 (defn save-db!
   [db-name data]
-  (let [[_db-name db-path] (sqlite-common-db/get-db-full-path (get-graphs-dir) db-name)]
+  (let [[_db-name db-path] (common-sqlite/get-db-full-path (get-graphs-dir) db-name)]
     (fs/writeFileSync db-path data)))
 
 (defn get-db
   [db-name]
   (let [_ (ensure-graph-dir! db-name)
-        [_db-name db-path] (sqlite-common-db/get-db-full-path (get-graphs-dir) db-name)]
+        [_db-name db-path] (common-sqlite/get-db-full-path (get-graphs-dir) db-name)]
     (when (fs/existsSync db-path)
       (fs/readFileSync db-path))))
 
@@ -37,7 +37,7 @@
 
 (defn unlink-graph!
   [repo]
-  (let [db-name (sqlite-common-db/sanitize-db-name repo)
+  (let [db-name (common-sqlite/sanitize-db-name repo)
         path (node-path/join (get-graphs-dir) db-name)
         unlinked (node-path/join (get-graphs-dir) unlinked-graphs-dir)
         new-path (node-path/join unlinked db-name)

src/electron/electron/handler.cljs

@@ -31,7 +31,7 @@
             [electron.window :as win]
             [goog.functions :refer [debounce]]
             [logseq.common.graph :as common-graph]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.sqlite.util :as sqlite-util]
             [promesa.core :as p]))
 
@@ -240,7 +240,7 @@
          (remove (fn [s] (= s db/unlinked-graphs-dir)))
          (map graph-name->path)
          (map (fn [s]
-                (if (string/starts-with? s sqlite-common-db/file-version-prefix)
+                (if (string/starts-with? s common-sqlite/file-version-prefix)
                   s
                   (str sqlite-util/db-version-prefix s)))))))
 
@@ -294,7 +294,7 @@
 (defmethod handle :deleteGraph [_window [_ graph graph-name _db-based?]]
   (when graph-name
     (db/unlink-graph! graph)
-    (let [old-transit-path (node-path/join (get-graphs-dir) (str (sqlite-common-db/sanitize-db-name graph) ".transit"))]
+    (let [old-transit-path (node-path/join (get-graphs-dir) (str (common-sqlite/sanitize-db-name graph) ".transit"))]
       (when (fs/existsSync old-transit-path)
         (fs/unlinkSync old-transit-path)))))
 

src/main/frontend/db/restore.cljs

@@ -1,12 +1,12 @@
 (ns frontend.db.restore
   "Fns for DB restore(from text or sqlite)"
   (:require [cljs-time.core :as t]
+            [datascript.core :as d]
             [frontend.db.conn :as db-conn]
             [frontend.persist-db :as persist-db]
             [frontend.state :as state]
             [frontend.undo-redo :as undo-redo]
             [logseq.db :as ldb]
-            [logseq.db.common.sqlite :as sqlite-common-db]
             [promesa.core :as p]))
 
 (defn restore-graph!
@@ -20,7 +20,7 @@
           _ (when (nil? schema)
               (throw (ex-info "No valid schema found when reloading db" {:repo repo})))
           conn (try
-                 (sqlite-common-db/restore-initial-data initial-data schema)
+                 (d/conn-from-datoms initial-data schema)
                  (catch :default e
                    (prn :error :restore-initial-data-failed
                         (ldb/write-transit-str {:schema schema

src/main/frontend/handler/db_based/rtc.cljs

@@ -10,7 +10,7 @@
             [frontend.util :as util]
             [lambdaisland.glogi :as log]
             [logseq.db :as ldb]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.shui.ui :as shui]
             [promesa.core :as p]))
 
@@ -19,7 +19,7 @@
   (p/do!
    (js/Promise. user-handler/task--ensure-id&access-token)
    (let [token (state/get-auth-id-token)
-         repo-name (sqlite-common-db/sanitize-db-name repo)]
+         repo-name (common-sqlite/sanitize-db-name repo)]
      (state/<invoke-db-worker :thread-api/rtc-async-upload-graph repo token repo-name))))
 
 (defn <rtc-delete-graph!

src/main/frontend/handler/export.cljs

@@ -22,7 +22,7 @@
    [goog.dom :as gdom]
    [lambdaisland.glogi :as log]
    [logseq.db :as ldb]
-   [logseq.db.common.sqlite :as sqlite-common-db]
+   [logseq.db.common.sqlite :as common-sqlite]
    [logseq.publishing.html :as publish-html]
    [promesa.core :as p])
   (:import
@@ -58,7 +58,7 @@
   [repo]
   (p/let [db-data (persist-db/<export-db repo {:return-data? true})
           filename "db.sqlite"
-          repo-name (sqlite-common-db/sanitize-db-name repo)
+          repo-name (common-sqlite/sanitize-db-name repo)
           assets (assets-handler/<get-all-assets)
           files (cons [filename db-data] assets)
           zipfile (zip/make-zip repo-name files repo)]
@@ -255,7 +255,7 @@
   (when (and repo (= repo (state/get-current-repo)))
     (when-let [backup-folder (ldb/get-key-value (db/get-db repo) :logseq.kv/graph-backup-folder)]
       (p/let [handle (idb/get-item (str "handle/" (js/btoa repo) "/" backup-folder))
-              repo-name (sqlite-common-db/sanitize-db-name repo)]
+              repo-name (common-sqlite/sanitize-db-name repo)]
         (if handle
           (->
            (p/let [graph-dir-handle (.getDirectoryHandle handle repo-name #js {:create true})

src/main/frontend/worker/db_worker.cljs

@@ -35,8 +35,9 @@
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.common.order :as db-order]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.common.view :as db-view]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
@@ -329,19 +330,19 @@
                                        :client-ops client-ops-db})
       (doseq [db' dbs]
         (enable-sqlite-wal-mode! db'))
-      (sqlite-common-db/create-kvs-table! db)
-      (when-not @*publishing? (sqlite-common-db/create-kvs-table! client-ops-db))
+      (common-sqlite/create-kvs-table! db)
+      (when-not @*publishing? (common-sqlite/create-kvs-table! client-ops-db))
       (db-migrate/migrate-sqlite-db db)
       (when-not @*publishing? (db-migrate/migrate-sqlite-db client-ops-db))
       (search/create-tables-and-triggers! search-db)
       (let [schema (ldb/get-schema repo)
-            conn (sqlite-common-db/get-storage-conn storage schema)
+            conn (common-sqlite/get-storage-conn storage schema)
             _ (db-fix/check-and-fix-schema! repo conn)
             _ (when datoms
                 (let [data (map (fn [datom]
                                   [:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
                   (d/transact! conn data {:initial-db? true})))
-            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn
+            client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn
                                                      client-ops-storage
                                                      client-op/schema-in-db))
             initial-data-exists? (when (nil? datoms)
@@ -492,7 +493,7 @@
                 id)]
       (some->> eid
                (d/pull @conn selector)
-               (sqlite-common-db/with-parent @conn)))))
+               (common-initial-data/with-parent @conn)))))
 
 (def ^:private *get-blocks-cache (volatile! (cache/lru-cache-factory {} :threshold 1000)))
 (def ^:private get-blocks-with-cache
@@ -506,7 +507,7 @@
      (when db
        (mapv (fn [{:keys [id opts]}]
                (let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)]
-                 (-> (sqlite-common-db/get-block-and-children db id' opts)
+                 (-> (common-initial-data/get-block-and-children db id' opts)
                      (assoc :id id)))) requests)))))
 
 (def-thread-api :thread-api/get-blocks
@@ -604,7 +605,7 @@
 (def-thread-api :thread-api/get-initial-data
   [repo]
   (when-let [conn (worker-state/get-datascript-conn repo)]
-    (sqlite-common-db/get-initial-data @conn)))
+    (common-initial-data/get-initial-data @conn)))
 
 (def-thread-api :thread-api/reset-db
   [repo db-transit]

src/main/frontend/worker/pipeline.cljs

@@ -12,7 +12,7 @@
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.sqlite.util :as sqlite-util]
@@ -233,7 +233,7 @@
                                 :db-after (:db-after result)))
                        tx-report)
           {:keys [pages blocks]} (ds-report/get-blocks-and-pages tx-report*)
-          _ (when (sqlite-common-db/local-file-based-graph? repo)
+          _ (when (common-sqlite/local-file-based-graph? repo)
               (let [page-ids (distinct (map :db/id pages))]
                 (doseq [page-id page-ids]
                   (when (d/entity @conn page-id)

src/main/frontend/worker/util.cljc

@@ -6,7 +6,7 @@
                      [goog.crypt :as crypt]
                      [goog.crypt.Hmac]
                      [goog.crypt.Sha256]
-                     [logseq.db.common.sqlite :as sqlite-common-db]
+                     [logseq.db.common.sqlite :as common-sqlite]
                      [frontend.common.file.util :as wfu])))
 
 ;; Copied from https://github.com/tonsky/datascript-todo
@@ -27,7 +27,7 @@
 
      (defn get-pool-name
        [graph-name]
-       (str "logseq-pool-" (sqlite-common-db/sanitize-db-name graph-name)))
+       (str "logseq-pool-" (common-sqlite/sanitize-db-name graph-name)))
 
      (defn- decode-username
        [username]