Merge branch 'feat/db' into feat/capacitor-new

.clj-kondo/config.edn

@@ -9,7 +9,7 @@
 
   all-frontend
   {:linters {:discouraged-namespace
-             {logseq.db.sqlite.cli {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}
+             {logseq.db.common.sqlite-cli {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}
               logseq.outliner.cli {:message "frontend should not depend on CLI namespace with sqlite3 dependency"}}}}
 
   ;; false positive with match/match and _
@@ -45,7 +45,7 @@
    rum.core/use-layout-effect! {:message "Use frontend.hooks/use-layout-effect! instead" :level :info}
    rum.core/use-callback {:message "Use frontend.hooks/use-callback instead" :level :info}}
   :unused-namespace {:level :warning
-                     :exclude [logseq.db.frontend.entity-plus]}
+                     :exclude [logseq.db.common.entity-plus]}
 
   ;; TODO:lint: Remove node-path excludes once we have a cleaner api
   :unresolved-var {:exclude [frontend.util/node-path.basename
@@ -173,10 +173,12 @@
              logseq.common.util.namespace ns-util
              logseq.common.util.page-ref page-ref
              logseq.db ldb
+             logseq.db.common.entity-plus entity-plus
              logseq.db.common.entity-util common-entity-util
+             logseq.db.common.initial-data common-initial-data
              logseq.db.common.order db-order
              logseq.db.common.property-util db-property-util
-             logseq.db.common.sqlite sqlite-common-db
+             logseq.db.common.sqlite common-sqlite
              logseq.db.common.view db-view
              logseq.db.file-based.rules file-rules
              logseq.db.file-based.schema file-schema
@@ -185,7 +187,6 @@
              logseq.db.frontend.content db-content
              logseq.db.frontend.db db-db
              logseq.db.frontend.db-ident db-ident
-             logseq.db.frontend.entity-plus entity-plus
              logseq.db.frontend.entity-util entity-util
              logseq.db.frontend.inputs db-inputs
              logseq.db.frontend.property db-property
@@ -195,7 +196,7 @@
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate
              logseq.db.sqlite.build sqlite-build
-             logseq.db.sqlite.cli sqlite-cli
+             logseq.db.common.sqlite-cli sqlite-cli
              logseq.db.sqlite.create-graph sqlite-create-graph
              logseq.db.sqlite.export sqlite-export
              logseq.db.sqlite.util sqlite-util

clj-e2e/dev/user.clj

@@ -1,9 +1,11 @@
 (ns user
   "fns used on repl"
   (:require [clojure.test :refer [run-tests run-test]]
+            [logseq.e2e.block :as b]
+            [logseq.e2e.commands-test]
             [logseq.e2e.config :as config]
-            [logseq.e2e.editor-test]
             [logseq.e2e.fixtures :as fixtures]
+            [logseq.e2e.keyboard :as k]
             [logseq.e2e.multi-tabs-test]
             [logseq.e2e.outliner-test]
             [logseq.e2e.rtc-basic-test]
@@ -23,10 +25,10 @@
   (some-> (get @*futures test-name) future-cancel)
   (swap! *futures dissoc test-name))
 
-(defn run-editor-test
+(defn run-commands-test
   []
-  (->> (future (run-tests 'logseq.e2e.editor-test))
-       (swap! *futures assoc :editor-test)))
+  (->> (future (run-tests 'logseq.e2e.commands-test))
+       (swap! *futures assoc :commands-test)))
 
 (defn run-outliner-test
   []
@@ -65,9 +67,14 @@
     (w/wait-for (first (util/get-edit-block-container))
                 {:state :detached}))
 
-  (run-tests 'logseq.e2e.editor-test
+  (run-tests 'logseq.e2e.commands-test
              'logseq.e2e.multi-tabs-test
              'logseq.e2e.outliner-test
              'logseq.e2e.rtc-basic-test)
+
+  (do
+    (reset! config/*headless true)
+    (dotimes [i 10]
+      (run-outliner-test)))
   ;;
   )

clj-e2e/src/logseq/e2e/assert.clj

@@ -29,3 +29,9 @@
   []
   ;; there's some blocks visible now
   (assert-is-visible "span.block-title-wrap"))
+
+(defn assert-editor-mode
+  []
+  (let [klass ".editor-wrapper textarea"
+        editor (w/-query klass)]
+    (w/wait-for editor)))

clj-e2e/src/logseq/e2e/block.clj

@@ -11,14 +11,30 @@
   (assert/assert-in-normal-mode?)
   (w/click (last (w/query ".ls-page-blocks .ls-block .block-content"))))
 
+(defn save-block
+  [text]
+  (w/fill ".editor-wrapper textarea" text))
+
 (defn new-block
   [title]
-  (k/enter)
-  (util/input title))
+  (let [editor? (util/get-editor)]
+    (when-not editor? (open-last-block))
+    (assert/assert-editor-mode)
+    (k/enter)
+    (save-block title)))
 
-(defn save-block
-  [text]
-  (util/input text))
+;; TODO: support tree
+(defn new-blocks
+  [titles]
+  (let [editor? (util/get-editor)]
+    (when-not editor? (open-last-block))
+    (assert/assert-editor-mode)
+    (let [value (util/get-edit-content)]
+      (if (string/blank? value)           ; empty block
+        (save-block (first titles))
+        (new-block (first titles))))
+    (doseq [title (rest titles)]
+      (new-block title))))
 
 (defn delete-blocks
   "Delete the current block if in editing mode, otherwise, delete all the selected blocks."
@@ -27,19 +43,6 @@
     (when editor (util/exit-edit))
     (k/backspace)))
 
-;; TODO: support tree
-(defn new-blocks
-  [titles]
-  (open-last-block)
-  (let [value (util/get-edit-content)]
-    (if (string/blank? value)           ; empty block
-      (do
-        (save-block (first titles))
-        (doseq [title (rest titles)]
-          (new-block title)))
-      (doseq [title titles]
-        (new-block title)))))
-
 (defn assert-blocks-visible
   "blocks - coll of :block/title"
   [blocks]

clj-e2e/src/logseq/e2e/keyboard.clj

@@ -8,5 +8,6 @@
 (def backspace #(press "Backspace"))
 (def tab #(press "Tab"))
 (def shift+tab #(press "Shift+Tab"))
+(def shift+enter #(press "Shift+Enter"))
 (def arrow-up #(press "ArrowUp"))
 (def arrow-down #(press "ArrowDown"))

clj-e2e/src/logseq/e2e/util.clj

@@ -161,3 +161,11 @@
   []
   (w/refresh)
   (assert/assert-graph-loaded?))
+
+(defn move-cursor-to-end
+  []
+  (k/press "ControlOrMeta+a" "ArrowRight"))
+
+(defn move-cursor-to-start
+  []
+  (k/press "ControlOrMeta+a" "ArrowLeft"))

clj-e2e/test/logseq/e2e/commands_test.clj

@@ -0,0 +1,122 @@
+(ns logseq.e2e.commands-test
+  (:require
+   [clojure.string :as string]
+   [clojure.test :refer [deftest testing is use-fixtures]]
+   [logseq.e2e.block :as b]
+   [logseq.e2e.fixtures :as fixtures]
+   [logseq.e2e.keyboard :as k]
+   [logseq.e2e.util :as util]
+   [wally.main :as w]))
+
+(use-fixtures :once fixtures/open-page)
+
+(use-fixtures :each fixtures/new-logseq-page)
+
+(deftest command-trigger-test
+  (testing "/command trigger popup"
+    (b/new-block "b2")
+    (util/type " /")
+    (w/wait-for ".ui__popover-content")
+    (is (some? (w/find-one-by-text "span" "Node reference")))
+    (k/backspace)
+    (w/wait-for-not-visible ".ui__popover-content")))
+
+(defn- input-command
+  [command-match]
+  (util/type "/")
+  (util/type command-match)
+  (w/wait-for ".ui__popover-content")
+  (k/enter))
+
+(deftest node-reference-test
+  (testing "Node reference"
+    (testing "Page reference"
+      (b/new-blocks ["b1" ""])
+      (input-command "Node eferen")
+      (util/type "Another page")
+      (k/enter)
+      (is (= "[[Another page]]" (util/get-edit-content)))
+      (util/exit-edit)
+      (is (= "Another page" (util/get-text "a.page-ref"))))
+    (testing "Block reference"
+      (b/new-block "")
+      (input-command "Node eferen")
+      (util/type "b1")
+      (util/wait-timeout 300)
+      (k/enter)
+      (is (string/includes? (util/get-edit-content) "[["))
+      (util/exit-edit)
+      (is (= "b1" (util/get-text ".block-ref"))))))
+
+(deftest link-test
+  (testing "/link"
+    (let [add-logseq-link (fn []
+                            (util/type "https://logseq.com")
+                            (k/tab)
+                            (util/type "Logseq")
+                            (k/tab)
+                            (k/enter))]
+      (b/new-block "")
+      (input-command "link")
+      (add-logseq-link)
+      (is (= "[Logseq](https://logseq.com)" (util/get-edit-content)))
+      (util/type " some content ")
+      (input-command "link")
+      (add-logseq-link)
+      (is (= (str "[Logseq](https://logseq.com)"
+                  " some content "
+                  "[Logseq](https://logseq.com)") (util/get-edit-content))))))
+
+(deftest link-image-test
+  (testing "/image link"
+    (b/new-block "")
+    (input-command "image link")
+    (util/type "https://logseq.com/test.png")
+    (k/tab)
+    (util/type "Logseq")
+    (k/tab)
+    (k/enter)
+    (is (= "![Logseq](https://logseq.com/test.png)" (util/get-edit-content)))))
+
+(deftest underline-test
+  (testing "/underline"
+    (b/new-block "")
+    (input-command "underline")
+    (is (= "<ins></ins>" (util/get-edit-content)))
+    (util/type "test")
+    (is (= "<ins>test</ins>" (util/get-edit-content)))
+    (util/move-cursor-to-end)))
+
+(deftest code-block-test
+  (testing "/code block"
+    (b/new-block "")
+    (input-command "code block")
+    (w/wait-for ".CodeMirror")
+    (util/wait-timeout 100)
+    ;; create another block
+    (k/shift+enter)))
+
+(deftest math-block-test
+  (testing "/math block"
+    (b/new-block "")
+    (input-command "math block")
+    (util/type "1 + 2 = 3")
+    (util/exit-edit)
+    (w/wait-for ".katex")))
+
+(deftest quote-test
+  (testing "/quote"
+    (b/new-block "")
+    (input-command "quote")
+    (w/wait-for "div[data-node-type='quote']")))
+
+(deftest headings-test
+  (testing "/heading"
+    (dotimes [i 6]
+      (let [heading (str "h" (inc i))
+            text (str heading " test ")]
+        (b/new-block text)
+        (input-command heading)
+        (is (= text (util/get-edit-content)))
+        (util/exit-edit)
+        (w/wait-for heading)))))

clj-e2e/test/logseq/e2e/editor_test.clj

@@ -1,44 +0,0 @@
-(ns logseq.e2e.editor-test
-  (:require
-   [clojure.string :as string]
-   [clojure.test :refer [deftest testing is use-fixtures]]
-   [logseq.e2e.block :as b]
-   [logseq.e2e.fixtures :as fixtures]
-   [logseq.e2e.keyboard :as k]
-   [logseq.e2e.util :as util]
-   [wally.main :as w]))
-
-(use-fixtures :once fixtures/open-page)
-
-(deftest commands-test
-  (testing "/command trigger popup"
-    (util/new-page "Test")
-    (b/save-block "b1")
-    (util/type " /")
-    (w/wait-for ".ui__popover-content")
-    (is (some? (w/find-one-by-text "span" "Node reference")))
-    (k/backspace)
-    (w/wait-for-not-visible ".ui__popover-content"))
-
-  (testing "Node reference"
-    (testing "Page reference"
-      (b/new-block "/")
-      (util/type "Node eferen")
-      (w/wait-for ".ui__popover-content")
-      (k/enter)
-      (util/type "Another page")
-      (k/enter)
-      (is (= "[[Another page]]" (util/get-edit-content)))
-      (util/exit-edit)
-      (is (= "Another page" (util/get-text "a.page-ref"))))
-    (testing "Block reference"
-      (b/new-block "/")
-      (util/type "Node eferen")
-      (w/wait-for ".ui__popover-content")
-      (k/enter)
-      (util/type "b1")
-      (util/wait-timeout 300)
-      (k/enter)
-      (is (string/includes? (util/get-edit-content) "[["))
-      (util/exit-edit)
-      (is (= "b1" (util/get-text ".block-ref"))))))

clj-e2e/test/logseq/e2e/fixtures.clj

@@ -1,7 +1,8 @@
 (ns logseq.e2e.fixtures
-  (:require [wally.main :as w]
-            [logseq.e2e.config :as config]
-            [logseq.e2e.playwright-page :as pw-page]))
+  (:require [logseq.e2e.config :as config]
+            [logseq.e2e.playwright-page :as pw-page]
+            [logseq.e2e.util :as util]
+            [wally.main :as w]))
 
 ;; TODO: save trace
 ;; TODO: parallel support
@@ -10,8 +11,7 @@
   (w/with-page-open
     (w/make-page {:headless (or headless @config/*headless)
                   :persistent false
-                  :slow-mo @config/*slow-mo
-                  })
+                  :slow-mo @config/*slow-mo})
     (w/navigate (str "http://localhost:" (or port @config/*port)))
     (f)))
 
@@ -55,3 +55,10 @@
     (binding [*pw-ctx* ctx]
       (f)
       (.close (.browser *pw-ctx*)))))
+
+(defonce *page-number (atom 0))
+
+(defn new-logseq-page
+  [f]
+  (util/new-page (str "page " (swap! *page-number inc)))
+  (f))

clj-e2e/test/logseq/e2e/outliner_test.clj

@@ -1,16 +1,16 @@
 (ns logseq.e2e.outliner-test
   (:require
    [clojure.test :refer [deftest testing is use-fixtures]]
+   [logseq.e2e.block :as b]
    [logseq.e2e.fixtures :as fixtures]
    [logseq.e2e.keyboard :as k]
-   [logseq.e2e.block :as b]
    [logseq.e2e.util :as util]
    [wally.main :as w]))
 
 (use-fixtures :once fixtures/open-page)
+(use-fixtures :each fixtures/new-logseq-page)
 
 (deftest create-test-page-and-insert-blocks
-  (util/new-page "p1")
   ;; a page block and a child block
   (is (= 2 (util/blocks-count)))
   (b/new-blocks ["first block" "second block"])
@@ -18,7 +18,6 @@
   (is (= 3 (util/blocks-count))))
 
 (deftest indent-and-outdent-test
-  (util/new-page "p2")
   (b/new-blocks ["b1" "b2"])
   (testing "simple indent and outdent"
     (util/indent)
@@ -44,7 +43,6 @@
       (is (and (= x2 x4) (= x3 x5) (< x2 x3))))))
 
 (deftest move-up-down-test
-  (util/new-page "p3")
   (b/new-blocks ["b1" "b2" "b3" "b4"])
   (util/repeat-keyboard 2 "Shift+ArrowUp")
   (let [contents (util/get-page-blocks-contents)]
@@ -58,16 +56,15 @@
 
 (deftest delete-test
   (testing "Delete blocks case 1"
-    (util/new-page "p4")
     (b/new-blocks ["b1" "b2" "b3" "b4"])
     (b/delete-blocks)                   ; delete b4
     (util/repeat-keyboard 2 "Shift+ArrowUp") ; select b3 and b2
     (b/delete-blocks)
     (is (= "b1" (util/get-edit-content)))
-    (is (= 1 (util/page-blocks-count))))
+    (is (= 1 (util/page-blocks-count)))))
 
+(deftest delete-test-with-children
   (testing "Delete block with its children"
-    (util/new-page "p5")
     (b/new-blocks ["b1" "b2" "b3" "b4"])
     (util/indent)
     (k/arrow-up)

deps/db/.carve/config.edn

@@ -1,7 +1,7 @@
 {:paths ["src"]
  :api-namespaces [logseq.db.common.sqlite
                   logseq.db.sqlite.util
-                  logseq.db.sqlite.cli
+                  logseq.db.common.sqlite-cli
                   logseq.db.frontend.property
                   logseq.db.frontend.property.build
                   logseq.db.common.property-util

deps/db/.carve/ignore

@@ -28,3 +28,9 @@ logseq.db.sqlite.export/build-import
 logseq.db.common.view/get-property-values
 ;; API
 logseq.db.common.view/get-view-data
+;; API
+logseq.db.common.initial-data/with-parent
+;; API
+logseq.db.common.initial-data/get-block-and-children
+;; API
+logseq.db.common.initial-data/get-initial-data

deps/db/.clj-kondo/config.edn

@@ -9,10 +9,11 @@
   :consistent-alias
   {:aliases {clojure.string string
              logseq.db ldb
+             logseq.db.common.entity-plus entity-plus
              logseq.db.common.entity-util common-entity-util
              logseq.db.common.order db-order
              logseq.db.common.property-util db-property-util
-             logseq.db.common.sqlite sqlite-common-db
+             logseq.db.common.sqlite common-sqlite
              logseq.db.common.view db-view
              logseq.db.frontend.content db-content
              logseq.db.frontend.class db-class
@@ -25,12 +26,12 @@
              logseq.db.file-based.rules file-rules
              logseq.db.file-based.schema file-schema
              logseq.db.file-based.entity-util file-entity-util
-             logseq.db.frontend.entity-plus entity-plus
              logseq.db.frontend.rules rules
              logseq.db.frontend.schema db-schema
              logseq.db.frontend.validate db-validate
              logseq.db.sqlite.build sqlite-build
-             logseq.db.sqlite.cli sqlite-cli
+             logseq.db.common.initial-data common-initial-data
+             logseq.db.common.sqlite-cli sqlite-cli
              logseq.db.sqlite.create-graph sqlite-create-graph
              logseq.db.sqlite.export sqlite-export
              logseq.db.sqlite.util sqlite-util}}}

deps/db/script/diff_graphs.cljs

@@ -7,7 +7,7 @@
             [clojure.pprint :as pprint]
             [clojure.string :as string]
             [logseq.common.config :as common-config]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.export :as sqlite-export]
             [nbb.core :as nbb]))
 

deps/db/script/dump_datoms.cljs

@@ -7,7 +7,7 @@
               ["path" :as path]
               [clojure.pprint :as pprint]
               [datascript.core :as d]
-              [logseq.db.sqlite.cli :as sqlite-cli]
+              [logseq.db.common.sqlite-cli :as sqlite-cli]
               [nbb.core :as nbb]))
 
 (defn read-graph

deps/db/script/export_graph.cljs

@@ -7,7 +7,7 @@
             [clojure.edn :as edn]
             [clojure.pprint :as pprint]
             [clojure.string :as string]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.export :as sqlite-export]
             [nbb.core :as nbb]))
 

deps/db/script/query.cljs

@@ -11,7 +11,7 @@
             [clojure.string :as string]
             [datascript.core :as d]
             [logseq.db.frontend.rules :as rules]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [nbb.core :as nbb]))
 
 (defn- sh

deps/db/script/validate_db.cljs

@@ -9,7 +9,7 @@
             [datascript.core :as d]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.validate :as db-validate]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [malli.core :as m]
             [malli.error :as me]
             [nbb.core :as nbb]))

deps/db/src/logseq/db.cljs

@@ -10,14 +10,16 @@
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
             [logseq.db.common.delete-blocks :as delete-blocks] ;; Load entity extensions
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.common.entity-util :as common-entity-util]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.initial-data :as common-initial-data]
+            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.db :as db-db]
-            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.rules :as rules]
+            [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.util :as sqlite-util])
   (:refer-clojure :exclude [object?]))
 
@@ -190,7 +192,7 @@
        (map first)
        (remove hidden?)))
 
-(def get-first-page-by-name sqlite-common-db/get-first-page-by-name)
+(def get-first-page-by-name common-initial-data/get-first-page-by-name)
 
 (def db-based-graph? entity-plus/db-based-graph?)
 
@@ -248,7 +250,7 @@
     (if-let [id (if (uuid? page-name-or-uuid) page-name-or-uuid
                     (parse-uuid page-name-or-uuid))]
       (d/entity db [:block/uuid id])
-      (d/entity db (sqlite-common-db/get-first-page-by-title db page-name-or-uuid)))))
+      (d/entity db (common-initial-data/get-first-page-by-title db page-name-or-uuid)))))
 
 (defn page-empty?
   "Whether a page is empty. Does it has a non-page block?
@@ -347,8 +349,8 @@
         (recur (:block/uuid parent) (conj parents' parent) (inc d))
         parents'))))
 
-(def get-block-children-ids sqlite-common-db/get-block-children-ids)
-(def get-block-children sqlite-common-db/get-block-children)
+(def get-block-children-ids common-initial-data/get-block-children-ids)
+(def get-block-children common-initial-data/get-block-children)
 
 (defn- get-sorted-page-block-ids
   [db page-id]
@@ -541,9 +543,9 @@
 (def get-class-ident-by-display-type db-db/get-class-ident-by-display-type)
 (def get-display-type-by-class-ident db-db/get-display-type-by-class-ident)
 
-(def get-recent-updated-pages sqlite-common-db/get-recent-updated-pages)
+(def get-recent-updated-pages common-initial-data/get-recent-updated-pages)
 
-(def get-latest-journals sqlite-common-db/get-latest-journals)
+(def get-latest-journals common-initial-data/get-latest-journals)
 
 (defn get-pages-relation
   [db with-journal?]
@@ -578,3 +580,10 @@
          :where
          [?page :block/tags ?tag]]
        db))
+
+(defn get-schema
+  "Returns schema for given repo"
+  [repo]
+  (if (sqlite-util/db-based-graph? repo)
+    db-schema/schema
+    file-schema/schema))

deps/db/src/logseq/db/common/delete_blocks.cljs

@@ -6,7 +6,7 @@
             [logseq.common.util.block-ref :as block-ref]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db.common.entity-util :as common-entity-util]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]))
 
 (defn- replace-ref-with-deleted-block-title

deps/db/src/logseq/db/frontend/entity_plus.cljc → deps/db/src/logseq/db/common/entity_plus.cljc

@@ -1,4 +1,4 @@
-(ns logseq.db.frontend.entity-plus
+(ns logseq.db.common.entity-plus
   "Add map ops such as assoc/dissoc to datascript Entity.
 
    NOTE: This doesn't work for nbb/sci yet because of https://github.com/babashka/sci/issues/639"
@@ -58,7 +58,7 @@
     ;; FIXME: Correct dependency ordering instead of resolve workaround
     #?(:org.babashka/nbb false
        :cljs (when-let [f (resolve 'frontend.common.missionary/background-task-running?)]
-               (f :logseq.db.frontend.entity-plus/reset-immutable-entities-cache!)))))
+               (f :logseq.db.common.entity-plus/reset-immutable-entities-cache!)))))
 
 (defn entity-memoized
   [db eid]

deps/db/src/logseq/db/common/initial_data.cljs

@@ -0,0 +1,321 @@
+(ns logseq.db.common.initial-data
+  "Provides db helper fns for graph initialization and lazy loading entities"
+  (:require [clojure.set :as set]
+            [datascript.core :as d]
+            [datascript.impl.entity :as de]
+            [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
+            [logseq.common.util.date-time :as date-time-util]
+            [logseq.db.common.entity-util :as common-entity-util]
+            [logseq.db.common.order :as db-order]
+            [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.frontend.entity-util :as entity-util]))
+
+(defn- get-pages-by-name
+  [db page-name]
+  (d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
+
+(defn get-first-page-by-name
+  "Return the oldest page's db id for :block/name"
+  [db page-name]
+  (when (and db (string? page-name))
+    (first (sort (map :e (get-pages-by-name db page-name))))))
+
+(defn get-first-page-by-title
+  "Return the oldest page's db id for :block/title"
+  [db page-name]
+  {:pre [(string? page-name)]}
+  (->> (d/datoms db :avet :block/title page-name)
+       (filter (fn [d]
+                 (let [e (d/entity db (:e d))]
+                   (common-entity-util/page? e))))
+       (map :e)
+       sort
+       first))
+
+(comment
+  (defn- get-built-in-files
+    [db]
+    (let [files ["logseq/config.edn"
+                 "logseq/custom.css"
+                 "logseq/custom.js"]]
+      (map #(d/pull db '[*] [:file/path %]) files))))
+
+(defn- get-all-files
+  [db]
+  (->> (d/datoms db :avet :file/path)
+       (mapcat (fn [e] (d/datoms db :eavt (:e e))))))
+
+(defn- with-block-refs
+  [db block]
+  (update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
+
+(defn with-parent
+  [db block]
+  (cond
+    (:block/page block)
+    (let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
+                   (select-keys e [:db/id :block/uuid]))]
+      (->>
+       (assoc block :block/parent parent)
+       (common-util/remove-nils-non-nested)
+       (with-block-refs db)))
+
+    :else
+    block))
+
+(comment
+  (defn- with-block-link
+    [db block]
+    (if (:block/link block)
+      (update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
+      block)))
+
+(defn- mark-block-fully-loaded
+  [b]
+  (assoc b :block.temp/fully-loaded? true))
+
+(comment
+  (defn- property-without-db-attrs
+    [property]
+    (dissoc property :db/index :db/valueType :db/cardinality))
+
+  (defn- property-with-values
+    [db block properties]
+    (when (entity-plus/db-based-graph? db)
+      (let [block (d/entity db (:db/id block))
+            property-vals (if properties
+                            (map block properties)
+                            (vals (:block/properties block)))]
+        (->> property-vals
+             (mapcat
+              (fn [property-values]
+                (let [values (->>
+                              (if (and (coll? property-values)
+                                       (map? (first property-values)))
+                                property-values
+                                #{property-values}))
+                      value-ids (when (every? map? values)
+                                  (->> (map :db/id values)
+                                       (filter (fn [id] (or (int? id) (keyword? id))))))
+                      value-blocks (->>
+                                    (when (seq value-ids)
+                                      (map
+                                       (fn [id] (d/pull db '[:db/id :block/uuid
+                                                             :block/name :block/title
+                                                             :logseq.property/value
+                                                             :block/tags :block/page
+                                                             :logseq.property/created-from-property] id))
+                                       value-ids))
+                                  ;; FIXME: why d/pull returns {:db/id db-ident} instead of {:db/id number-eid}?
+                                    (keep (fn [block]
+                                            (let [from-property-id (get-in block [:logseq.property/created-from-property :db/id])]
+                                              (if (keyword? from-property-id)
+                                                (assoc-in block [:logseq.property/created-from-property :db/id] (:db/id (d/entity db from-property-id)))
+                                                block)))))]
+                  value-blocks))))))))
+
+(defn get-block-children-ids
+  "Returns children UUIDs"
+  [db block-uuid]
+  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
+    (let [seen   (volatile! [])]
+      (loop [steps          100      ;check result every 100 steps
+             eids-to-expand [eid]]
+        (when (seq eids-to-expand)
+          (let [eids-to-expand*
+                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
+                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
+            (when (and (zero? steps)
+                       (seq (set/intersection (set @seen) (set uuids-to-add))))
+              (throw (ex-info "bad outliner data, need to re-index to fix"
+                              {:seen @seen :eids-to-expand eids-to-expand})))
+            (vswap! seen (partial apply conj) uuids-to-add)
+            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
+      @seen)))
+
+(defn get-block-children
+  "Including nested children."
+  [db block-uuid]
+  (let [ids (get-block-children-ids db block-uuid)]
+    (when (seq ids)
+      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
+        (d/pull-many db '[*] ids')))))
+
+(defn- with-raw-title
+  [m entity]
+  (if-let [raw-title (:block/raw-title entity)]
+    (assoc m :block/title raw-title)
+    m))
+
+(defn- entity->map
+  [entity]
+  (-> (into {} entity)
+      (with-raw-title entity)
+      (assoc :db/id (:db/id entity))))
+
+(defn ^:large-vars/cleanup-todo get-block-and-children
+  [db id {:keys [children? children-only? nested-children? properties children-props]}]
+  (let [block (d/entity db (if (uuid? id)
+                             [:block/uuid id]
+                             id))
+        block-refs-count? (some #{:block.temp/refs-count} properties)
+        whiteboard? (common-entity-util/whiteboard? block)]
+    (when block
+      (let [children (when (or children? children-only?)
+                       (let [page? (common-entity-util/page? block)
+                             children (->>
+                                       (cond
+                                         (and nested-children? (not page?))
+                                         (get-block-children db (:block/uuid block))
+                                         nested-children?
+                                         (:block/_page block)
+                                         :else
+                                         (let [short-page? (when page?
+                                                             (<= (count (:block/_page block)) 100))]
+                                           (if short-page?
+                                             (:block/_page block)
+                                             (:block/_parent block))))
+                                       (remove (fn [e] (or (:logseq.property/created-from-property e)
+                                                           (:block/closed-value-property e)))))
+                             children-props (if whiteboard?
+                                              '[*]
+                                              (or children-props
+                                                  [:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
+                                                   ;; pre-loading feature-related properties to avoid UI refreshing
+                                                   :logseq.task/status :logseq.property.node/display-type]))]
+                         (map
+                          (fn [block]
+                            (if (= children-props '[*])
+                              (entity->map block)
+                              (-> (select-keys block children-props)
+                                  (with-raw-title block))))
+                          children)))]
+        (if children-only?
+          {:children children}
+          (let [block' (if (seq properties)
+                         (-> (select-keys block properties)
+                             (with-raw-title block)
+                             (assoc :db/id (:db/id block)))
+                         (entity->map block))
+                block' (cond->
+                        (mark-block-fully-loaded block')
+                         true
+                         (update-vals (fn [v]
+                                        (cond
+                                          (de/entity? v)
+                                          (entity->map v)
+                                          (and (coll? v) (every? de/entity? v))
+                                          (map entity->map v)
+
+                                          :else
+                                          v)))
+                         block-refs-count?
+                         (assoc :block.temp/refs-count (count (:block/_refs block))))]
+            (cond->
+             {:block block'}
+              children?
+              (assoc :children children))))))))
+
+(defn get-latest-journals
+  [db]
+  (let [today (date-time-util/date->int (js/Date.))]
+    (->> (d/datoms db :avet :block/journal-day)
+         vec
+         rseq
+         (keep (fn [d]
+                 (and (<= (:v d) today)
+                      (let [e (d/entity db (:e d))]
+                        (when (and (common-entity-util/journal? e) (:db/id e))
+                          e))))))))
+
+(defn- get-structured-datoms
+  [db]
+  (let [class-property-id (:db/id (d/entity db :logseq.class/Property))]
+    (->> (concat
+          (d/datoms db :avet :block/tags :logseq.class/Tag)
+          (d/datoms db :avet :block/tags :logseq.class/Property)
+          (d/datoms db :avet :block/closed-value-property))
+         (mapcat (fn [d]
+                   (let [block-datoms (d/datoms db :eavt (:e d))
+                         property-desc-datoms (when (= (:v d) class-property-id)
+                                                (when-let [desc (:logseq.property/description (d/entity db (:e d)))]
+                                                  (d/datoms db :eavt (:db/id desc))))]
+                     (if property-desc-datoms
+                       (concat block-datoms property-desc-datoms)
+                       block-datoms)))))))
+
+(defn- get-favorites
+  "Favorites page and its blocks"
+  [db]
+  (let [page-id (get-first-page-by-name db common-config/favorites-page-name)
+        block (d/entity db page-id)
+        children (:block/_page block)]
+    (when block
+      (concat (d/datoms db :eavt (:db/id block))
+              (->> (keep :block/link children)
+                   (mapcat (fn [l]
+                             (d/datoms db :eavt (:db/id l)))))
+              (mapcat (fn [child]
+                        (d/datoms db :eavt (:db/id child)))
+                      children)))))
+
+(defn- get-views-data
+  [db]
+  (let [page-id (get-first-page-by-name db common-config/views-page-name)
+        children (when page-id (:block/_parent (d/entity db page-id)))]
+    (when (seq children)
+      (into
+       (mapcat (fn [b] (d/datoms db :eavt (:db/id b)))
+               children)
+       (d/datoms db :eavt page-id)))))
+
+(defn get-recent-updated-pages
+  [db]
+  (some->>
+   (d/datoms db :avet :block/updated-at)
+   rseq
+   (keep (fn [datom]
+           (let [e (d/entity db (:e datom))]
+             (when (and (common-entity-util/page? e) (not (entity-util/hidden? e)))
+               e))))
+   (take 30)))
+
+(defn get-initial-data
+  "Returns current database schema and initial data.
+   NOTE: This fn is called by DB and file graphs"
+  [db]
+  (let [db-graph? (entity-plus/db-based-graph? db)
+        _ (when db-graph?
+            (reset! db-order/*max-key (db-order/get-max-order db)))
+        schema (:schema db)
+        idents (mapcat (fn [id]
+                         (when-let [e (d/entity db id)]
+                           (d/datoms db :eavt (:db/id e))))
+                       [:logseq.kv/db-type
+                        :logseq.kv/schema-version
+                        :logseq.kv/graph-uuid
+                        :logseq.kv/latest-code-lang
+                        :logseq.kv/graph-backup-folder
+                        :logseq.property/empty-placeholder])
+        favorites (when db-graph? (get-favorites db))
+        views (when db-graph? (get-views-data db))
+        all-files (get-all-files db)
+        structured-datoms (when db-graph?
+                            (get-structured-datoms db))
+        recent-updated-pages (let [pages (get-recent-updated-pages db)]
+                               (mapcat (fn [p] (d/datoms db :eavt (:db/id p))) pages))
+        pages-datoms (let [contents-id (get-first-page-by-title db "Contents")
+                           views-id (get-first-page-by-title db common-config/views-page-name)]
+                       (mapcat #(d/datoms db :eavt %)
+                               (remove nil? [contents-id views-id])))
+        data (distinct
+              (concat idents
+                      structured-datoms
+                      favorites
+                      recent-updated-pages
+                      views
+                      all-files
+                      pages-datoms))]
+    {:schema schema
+     :initial-data data}))

deps/db/src/logseq/db/common/property_util.cljs

@@ -1,7 +1,7 @@
 (ns logseq.db.common.property-util
   "Property related util fns. Fns used in both DB and file graphs should go here"
   (:require [datascript.core :as d]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
             [logseq.db.sqlite.util :as sqlite-util]))

deps/db/src/logseq/db/common/sqlite.cljs

@@ -1,343 +1,11 @@
 (ns logseq.db.common.sqlite
-  "Provides common sqlite db fns for file and DB graphs. These fns work on
+  "Provides common sqlite util fns for file and DB graphs. These fns work on
   browser and node"
   (:require ["path" :as node-path]
-            [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
-            [datascript.impl.entity :as de]
-            [logseq.common.config :as common-config]
-            [logseq.common.util :as common-util]
-            [logseq.common.util.date-time :as date-time-util]
-            [logseq.db.common.entity-util :as common-entity-util]
-            [logseq.db.common.order :as db-order]
-            [logseq.db.frontend.entity-plus :as entity-plus]
-            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.sqlite.util :as sqlite-util]))
 
-(defn- get-pages-by-name
-  [db page-name]
-  (d/datoms db :avet :block/name (common-util/page-name-sanity-lc page-name)))
-
-(defn get-first-page-by-name
-  "Return the oldest page's db id for :block/name"
-  [db page-name]
-  (when (and db (string? page-name))
-    (first (sort (map :e (get-pages-by-name db page-name))))))
-
-(defn get-first-page-by-title
-  "Return the oldest page's db id for :block/title"
-  [db page-name]
-  {:pre [(string? page-name)]}
-  (->> (d/datoms db :avet :block/title page-name)
-       (filter (fn [d]
-                 (let [e (d/entity db (:e d))]
-                   (common-entity-util/page? e))))
-       (map :e)
-       sort
-       first))
-
-(comment
-  (defn- get-built-in-files
-    [db]
-    (let [files ["logseq/config.edn"
-                 "logseq/custom.css"
-                 "logseq/custom.js"]]
-      (map #(d/pull db '[*] [:file/path %]) files))))
-
-(defn get-all-files
-  [db]
-  (->> (d/datoms db :avet :file/path)
-       (mapcat (fn [e] (d/datoms db :eavt (:e e))))))
-
-(defn- with-block-refs
-  [db block]
-  (update block :block/refs (fn [refs] (map (fn [ref] (d/pull db '[*] (:db/id ref))) refs))))
-
-(defn with-parent
-  [db block]
-  (cond
-    (:block/page block)
-    (let [parent (when-let [e (d/entity db (:db/id (:block/parent block)))]
-                   (select-keys e [:db/id :block/uuid]))]
-      (->>
-       (assoc block :block/parent parent)
-       (common-util/remove-nils-non-nested)
-       (with-block-refs db)))
-
-    :else
-    block))
-
-(comment
-  (defn- with-block-link
-    [db block]
-    (if (:block/link block)
-      (update block :block/link (fn [link] (d/pull db '[*] (:db/id link))))
-      block)))
-
-(defn- mark-block-fully-loaded
-  [b]
-  (assoc b :block.temp/fully-loaded? true))
-
-(comment
-  (defn- property-without-db-attrs
-    [property]
-    (dissoc property :db/index :db/valueType :db/cardinality))
-
-  (defn- property-with-values
-    [db block properties]
-    (when (entity-plus/db-based-graph? db)
-      (let [block (d/entity db (:db/id block))
-            property-vals (if properties
-                            (map block properties)
-                            (vals (:block/properties block)))]
-        (->> property-vals
-             (mapcat
-              (fn [property-values]
-                (let [values (->>
-                              (if (and (coll? property-values)
-                                       (map? (first property-values)))
-                                property-values
-                                #{property-values}))
-                      value-ids (when (every? map? values)
-                                  (->> (map :db/id values)
-                                       (filter (fn [id] (or (int? id) (keyword? id))))))
-                      value-blocks (->>
-                                    (when (seq value-ids)
-                                      (map
-                                       (fn [id] (d/pull db '[:db/id :block/uuid
-                                                             :block/name :block/title
-                                                             :logseq.property/value
-                                                             :block/tags :block/page
-                                                             :logseq.property/created-from-property] id))
-                                       value-ids))
-                                  ;; FIXME: why d/pull returns {:db/id db-ident} instead of {:db/id number-eid}?
-                                    (keep (fn [block]
-                                            (let [from-property-id (get-in block [:logseq.property/created-from-property :db/id])]
-                                              (if (keyword? from-property-id)
-                                                (assoc-in block [:logseq.property/created-from-property :db/id] (:db/id (d/entity db from-property-id)))
-                                                block)))))]
-                  value-blocks))))))))
-
-(defn get-block-children-ids
-  "Returns children UUIDs"
-  [db block-uuid]
-  (when-let [eid (:db/id (d/entity db [:block/uuid block-uuid]))]
-    (let [seen   (volatile! [])]
-      (loop [steps          100      ;check result every 100 steps
-             eids-to-expand [eid]]
-        (when (seq eids-to-expand)
-          (let [eids-to-expand*
-                (mapcat (fn [eid] (map first (d/datoms db :avet :block/parent eid))) eids-to-expand)
-                uuids-to-add (remove nil? (map #(:block/uuid (d/entity db %)) eids-to-expand*))]
-            (when (and (zero? steps)
-                       (seq (set/intersection (set @seen) (set uuids-to-add))))
-              (throw (ex-info "bad outliner data, need to re-index to fix"
-                              {:seen @seen :eids-to-expand eids-to-expand})))
-            (vswap! seen (partial apply conj) uuids-to-add)
-            (recur (if (zero? steps) 100 (dec steps)) eids-to-expand*))))
-      @seen)))
-
-(defn get-block-children
-  "Including nested children."
-  [db block-uuid]
-  (let [ids (get-block-children-ids db block-uuid)]
-    (when (seq ids)
-      (let [ids' (map (fn [id] [:block/uuid id]) ids)]
-        (d/pull-many db '[*] ids')))))
-
-(defn- with-raw-title
-  [m entity]
-  (if-let [raw-title (:block/raw-title entity)]
-    (assoc m :block/title raw-title)
-    m))
-
-(defn- entity->map
-  [entity]
-  (-> (into {} entity)
-      (with-raw-title entity)
-      (assoc :db/id (:db/id entity))))
-
-(defn ^:large-vars/cleanup-todo get-block-and-children
-  [db id {:keys [children? children-only? nested-children? properties children-props]}]
-  (let [block (d/entity db (if (uuid? id)
-                             [:block/uuid id]
-                             id))
-        block-refs-count? (some #{:block.temp/refs-count} properties)
-        whiteboard? (common-entity-util/whiteboard? block)]
-    (when block
-      (let [children (when (or children? children-only?)
-                       (let [page? (common-entity-util/page? block)
-                             children (->>
-                                       (cond
-                                         (and nested-children? (not page?))
-                                         (get-block-children db (:block/uuid block))
-                                         nested-children?
-                                         (:block/_page block)
-                                         :else
-                                         (let [short-page? (when page?
-                                                             (<= (count (:block/_page block)) 100))]
-                                           (if short-page?
-                                             (:block/_page block)
-                                             (:block/_parent block))))
-                                       (remove (fn [e] (or (:logseq.property/created-from-property e)
-                                                           (:block/closed-value-property e)))))
-                             children-props (if whiteboard?
-                                              '[*]
-                                              (or children-props
-                                                  [:db/id :block/uuid :block/parent :block/order :block/collapsed? :block/title
-                                                   ;; pre-loading feature-related properties to avoid UI refreshing
-                                                   :logseq.task/status :logseq.property.node/display-type]))]
-                         (map
-                          (fn [block]
-                            (if (= children-props '[*])
-                              (entity->map block)
-                              (-> (select-keys block children-props)
-                                  (with-raw-title block))))
-                          children)))]
-        (if children-only?
-          {:children children}
-          (let [block' (if (seq properties)
-                         (-> (select-keys block properties)
-                             (with-raw-title block)
-                             (assoc :db/id (:db/id block)))
-                         (entity->map block))
-                block' (cond->
-                        (mark-block-fully-loaded block')
-                         true
-                         (update-vals (fn [v]
-                                        (cond
-                                          (de/entity? v)
-                                          (entity->map v)
-                                          (and (coll? v) (every? de/entity? v))
-                                          (map entity->map v)
-
-                                          :else
-                                          v)))
-                         block-refs-count?
-                         (assoc :block.temp/refs-count (count (:block/_refs block))))]
-            (cond->
-             {:block block'}
-              children?
-              (assoc :children children))))))))
-
-(defn get-latest-journals
-  [db]
-  (let [today (date-time-util/date->int (js/Date.))]
-    (->> (d/datoms db :avet :block/journal-day)
-         vec
-         rseq
-         (keep (fn [d]
-                 (and (<= (:v d) today)
-                      (let [e (d/entity db (:e d))]
-                        (when (and (common-entity-util/journal? e) (:db/id e))
-                          e))))))))
-
-(defn get-page->refs-count
-  [db]
-  (let [datoms (d/datoms db :avet :block/name)]
-    (->>
-     (map (fn [d]
-            [(:e d)
-             (count (:block/_refs (d/entity db (:e d))))]) datoms)
-     (into {}))))
-
-(defn get-structured-datoms
-  [db]
-  (let [class-property-id (:db/id (d/entity db :logseq.class/Property))]
-    (->> (concat
-          (d/datoms db :avet :block/tags :logseq.class/Tag)
-          (d/datoms db :avet :block/tags :logseq.class/Property)
-          (d/datoms db :avet :block/closed-value-property))
-         (mapcat (fn [d]
-                   (let [block-datoms (d/datoms db :eavt (:e d))
-                         property-desc-datoms (when (= (:v d) class-property-id)
-                                                (when-let [desc (:logseq.property/description (d/entity db (:e d)))]
-                                                  (d/datoms db :eavt (:db/id desc))))]
-                     (if property-desc-datoms
-                       (concat block-datoms property-desc-datoms)
-                       block-datoms)))))))
-
-(defn get-favorites
-  "Favorites page and its blocks"
-  [db]
-  (let [page-id (get-first-page-by-name db common-config/favorites-page-name)
-        block (d/entity db page-id)
-        children (:block/_page block)]
-    (when block
-      (concat (d/datoms db :eavt (:db/id block))
-              (->> (keep :block/link children)
-                   (mapcat (fn [l]
-                             (d/datoms db :eavt (:db/id l)))))
-              (mapcat (fn [child]
-                        (d/datoms db :eavt (:db/id child)))
-                      children)))))
-
-(defn get-views-data
-  [db]
-  (let [page-id (get-first-page-by-name db common-config/views-page-name)
-        children (when page-id (:block/_parent (d/entity db page-id)))]
-    (when (seq children)
-      (into
-       (mapcat (fn [b] (d/datoms db :eavt (:db/id b)))
-               children)
-       (d/datoms db :eavt page-id)))))
-
-(defn get-recent-updated-pages
-  [db]
-  (some->>
-   (d/datoms db :avet :block/updated-at)
-   rseq
-   (keep (fn [datom]
-           (let [e (d/entity db (:e datom))]
-             (when (and (common-entity-util/page? e) (not (entity-util/hidden? e)))
-               e))))
-   (take 30)))
-
-(defn get-initial-data
-  "Returns current database schema and initial data.
-   NOTE: This fn is called by DB and file graphs"
-  [db]
-  (let [db-graph? (entity-plus/db-based-graph? db)
-        _ (when db-graph?
-            (reset! db-order/*max-key (db-order/get-max-order db)))
-        schema (:schema db)
-        idents (mapcat (fn [id]
-                         (when-let [e (d/entity db id)]
-                           (d/datoms db :eavt (:db/id e))))
-                       [:logseq.kv/db-type
-                        :logseq.kv/schema-version
-                        :logseq.kv/graph-uuid
-                        :logseq.kv/latest-code-lang
-                        :logseq.kv/graph-backup-folder
-                        :logseq.property/empty-placeholder])
-        favorites (when db-graph? (get-favorites db))
-        views (when db-graph? (get-views-data db))
-        all-files (get-all-files db)
-        structured-datoms (when db-graph?
-                            (get-structured-datoms db))
-        recent-updated-pages (let [pages (get-recent-updated-pages db)]
-                               (mapcat (fn [p] (d/datoms db :eavt (:db/id p))) pages))
-        pages-datoms (let [contents-id (get-first-page-by-title db "Contents")
-                           views-id (get-first-page-by-title db common-config/views-page-name)]
-                       (mapcat #(d/datoms db :eavt %)
-                               (remove nil? [contents-id views-id])))
-        data (distinct
-              (concat idents
-                      structured-datoms
-                      favorites
-                      recent-updated-pages
-                      views
-                      all-files
-                      pages-datoms))]
-    {:schema schema
-     :initial-data data}))
-
-(defn restore-initial-data
-  "Given initial Datascript datoms and schema, returns a datascript connection"
-  [data schema]
-  (d/conn-from-datoms data schema))
-
 (defn create-kvs-table!
   "Creates a sqlite table for use with datascript.storage if one doesn't exist"
   [sqlite-db]
@@ -349,9 +17,16 @@
   (or (d/restore-conn storage)
       (d/create-conn schema {:storage storage})))
 
+(defonce file-version-prefix "logseq_local_")
+
+(defn local-file-based-graph?
+  [s]
+  (and (string? s)
+       (string/starts-with? s file-version-prefix)))
+
 (defn sanitize-db-name
   [db-name]
-  (if (string/starts-with? db-name sqlite-util/file-version-prefix)
+  (if (string/starts-with? db-name file-version-prefix)
     (-> db-name
         (string/replace ":" "+3A+")
         (string/replace "/" "++"))

deps/db/src/logseq/db/sqlite/cli.cljs → deps/db/src/logseq/db/common/sqlite_cli.cljs

@@ -1,4 +1,4 @@
-(ns ^:node-only logseq.db.sqlite.cli
+(ns ^:node-only logseq.db.common.sqlite-cli
   "Primary ns to interact with DB files for DB and file graphs with node.js based CLIs"
   (:require ["better-sqlite3" :as sqlite3]
             ["fs" :as fs]
@@ -7,7 +7,7 @@
             ;; FIXME: datascript.core has to come before datascript.storage or else nbb fails
             [datascript.core]
             [datascript.storage :refer [IStorage]]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.util :as sqlite-util]))
@@ -87,13 +87,13 @@
   needed sqlite tables if not created and returns a datascript connection that's
   connected to the sqlite db"
   [graphs-dir db-name]
-  (let [[_db-sanitized-name db-full-path] (sqlite-common-db/get-db-full-path graphs-dir db-name)
+  (let [[_db-sanitized-name db-full-path] (common-sqlite/get-db-full-path graphs-dir db-name)
         db (new sqlite db-full-path nil)
         ;; For both desktop and CLI, only file graphs have db-name that indicate their db type
-        schema (if (sqlite-util/local-file-based-graph? db-name)
+        schema (if (common-sqlite/local-file-based-graph? db-name)
                  file-schema/schema
                  db-schema/schema)]
-    (sqlite-common-db/create-kvs-table! db)
+    (common-sqlite/create-kvs-table! db)
     (let [storage (new-sqlite-storage db)
-          conn (sqlite-common-db/get-storage-conn storage schema)]
+          conn (common-sqlite/get-storage-conn storage schema)]
       conn)))

deps/db/src/logseq/db/common/view.cljs

@@ -9,7 +9,7 @@
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]

deps/db/src/logseq/db/file_based/schema.cljs

@@ -32,9 +32,8 @@
    :block/path-refs {:db/valueType   :db.type/ref
                      :db/cardinality :db.cardinality/many}
 
-   ;; tags are structured classes
-   :block/tags {:db/valueType :db.type/ref
-                :db/cardinality :db.cardinality/many}
+  :block/tags {:db/valueType :db.type/ref
+               :db/cardinality :db.cardinality/many}
 
    ;; which block this block links to, used for tag, embeds
    :block/link {:db/valueType :db.type/ref

deps/db/src/logseq/db/frontend/malli_schema.cljs

@@ -5,7 +5,7 @@
             [datascript.core :as d]
             [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]

deps/db/src/logseq/db/frontend/schema.cljs

@@ -61,7 +61,7 @@
     :else (throw (ex-info "Not a schema-version" {:data schema-version}))))
 
 (def schema
-  "Schema for DB graphs"
+  "Schema for DB graphs. :block/tags are classes in this schema"
   (merge
    (apply dissoc file-schema/schema file-schema/file-only-attributes)
    {:block/name {:db/index true}        ; remove db/unique for :block/name

deps/db/src/logseq/db/sqlite/create_graph.cljs

@@ -170,7 +170,7 @@
   (build-initial-classes* db-class/built-in-classes db-ident->properties))
 
 (defn build-initial-views
-  "Builds initial blocks used for storing views. Used by db and file graphs"
+  "Builds initial blocks used for storing views"
   []
   (let [page-id (common-uuid/gen-uuid :builtin-block-uuid common-config/views-page-name)]
     [(sqlite-util/block-with-timestamps

deps/db/src/logseq/db/sqlite/export.cljs

@@ -10,7 +10,7 @@
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.db :as db-db]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.build :as sqlite-build]

deps/db/src/logseq/db/sqlite/util.cljs

@@ -9,13 +9,10 @@
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
             [logseq.db.common.order :as db-order]
-            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.property :as db-property]
-            [logseq.db.frontend.property.type :as db-property-type]
-            [logseq.db.frontend.schema :as db-schema]))
+            [logseq.db.frontend.property.type :as db-property-type]))
 
 (defonce db-version-prefix "logseq_db_")
-(defonce file-version-prefix "logseq_local_")
 
 (def ^:private write-handlers (cljs-bean.transit/writer-handlers))
 (def ^:private read-handlers {})
@@ -59,18 +56,6 @@
   (when graph-name
     (string/starts-with? graph-name db-version-prefix)))
 
-(defn local-file-based-graph?
-  [s]
-  (and (string? s)
-       (string/starts-with? s file-version-prefix)))
-
-(defn get-schema
-  "Returns schema for given repo"
-  [repo]
-  (if (db-based-graph? repo)
-    db-schema/schema
-    file-schema/schema))
-
 (def block-with-timestamps common-util/block-with-timestamps)
 
 (defn build-new-property

deps/db/src/logseq/db/test/helper.cljs

@@ -2,7 +2,7 @@
   "Main ns for providing test fns for DB graphs"
   (:require [datascript.core :as d]
             [datascript.impl.entity :as de]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.build :as sqlite-build]

deps/db/test/logseq/db/common/sqlite_test.cljs → deps/db/test/logseq/db/common/initial_data_test.cljs

@@ -1,13 +1,13 @@
-(ns logseq.db.common.sqlite-test
+(ns logseq.db.common.initial-data-test
   "This ns is the only one to test against file based datascript connections.
    These are useful integration tests"
   (:require ["fs" :as fs]
             ["path" :as node-path]
             [cljs.test :refer [deftest async use-fixtures is testing]]
             [datascript.core :as d]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.sqlite.build :as sqlite-build]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.test.helper :as db-test]))
 
@@ -35,8 +35,8 @@
                    :file/content "{:foo :bar}"}]
           _ (d/transact! conn* blocks)
           ;; Simulate getting data from sqlite and restoring it for frontend
-          {:keys [schema initial-data]} (sqlite-common-db/get-initial-data @conn*)
-          conn (sqlite-common-db/restore-initial-data initial-data schema)]
+          {:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
+          conn (d/conn-from-datoms initial-data schema)]
       (is (= blocks
              (->> @conn
                   (d/q '[:find (pull ?b [:block/uuid :file/path :file/content]) :where [?b :file/content]])
@@ -54,7 +54,7 @@
             :blocks [{:block/title "b1"}]}]})
         _ (d/transact! conn* init-tx)
           ;; Simulate getting data from sqlite and restoring it for frontend
-        {:keys [schema initial-data]} (sqlite-common-db/get-initial-data @conn*)
-        conn (sqlite-common-db/restore-initial-data initial-data schema)]
+        {:keys [schema initial-data]} (common-initial-data/get-initial-data @conn*)
+        conn (d/conn-from-datoms initial-data schema)]
     (is (some? (db-test/find-page-by-title @conn "page1"))
         "Restores recently updated page")))

deps/graph-parser/test/logseq/graph_parser/exporter_test.cljs

@@ -10,7 +10,7 @@
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db :as ldb]
             [logseq.db.frontend.content :as db-content]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.rules :as rules]
             [logseq.db.frontend.validate :as db-validate]

deps/outliner/script/transact.cljs

@@ -6,7 +6,7 @@
             [clojure.string :as string]
             [datascript.core :as d]
             [logseq.db.frontend.rules :as rules]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.outliner.db-pipeline :as db-pipeline]
             [nbb.core :as nbb]))
 

deps/outliner/src/logseq/outliner/cli.cljs

@@ -4,7 +4,7 @@
             [datascript.core :as d]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.build :as sqlite-build]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.outliner.db-pipeline :as db-pipeline]
             ["fs" :as fs]
             ["path" :as node-path]))

deps/outliner/src/logseq/outliner/pipeline.cljs

@@ -6,7 +6,7 @@
             [logseq.common.util.date-time :as date-time-util]
             [logseq.db :as ldb]
             [logseq.db.frontend.content :as db-content]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
             [logseq.outliner.datascript-report :as ds-report]))
 

deps/outliner/src/logseq/outliner/property.cljs

@@ -8,7 +8,7 @@
             [logseq.db :as ldb]
             [logseq.db.common.order :as db-order]
             [logseq.db.frontend.db-ident :as db-ident]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.property :as db-property]

deps/outliner/test/logseq/outliner/validate_test.cljs

@@ -1,7 +1,7 @@
 (ns logseq.outliner.validate-test
   (:require [cljs.test :refer [are deftest is testing]]
             [datascript.core :as d]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.test.helper :as db-test]
             [logseq.outliner.validate :as outliner-validate]))
 

deps/publishing/script/publishing.cljs

@@ -4,7 +4,7 @@
             ["path" :as node-path]
             [clojure.edn :as edn]
             [datascript.core :as d]
-            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.common.sqlite-cli :as sqlite-cli]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.cli :as gp-cli]
             [logseq.publishing :as publishing]

deps/publishing/src/logseq/publishing/db.cljs

@@ -3,7 +3,7 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.rules :as rules]))
 

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -61,7 +61,8 @@
 
 (def file-graph-paths
   "Paths _only_ for file graphs"
-  ["deps/graph-parser/src/logseq/graph_parser/db.cljs"
+  ["deps/db/src/logseq/db/file_based"
+   "deps/graph-parser/src/logseq/graph_parser/db.cljs"
    "deps/graph-parser/src/logseq/graph_parser/extract.cljc"
    "deps/graph-parser/src/logseq/graph_parser/property.cljs"
    "deps/graph-parser/src/logseq/graph_parser.cljs"
@@ -98,8 +99,7 @@
 
 (defn- validate-multi-graph-fns-not-in-file-or-db
   []
-  ;; TODO: Lint `(db-based-graph?` when db.frontend.entity-plus is split into separate graph contexts
-  (let [multi-graph-fns ["/db-based-graph\\?"
+  (let [multi-graph-fns ["/db-based-graph\\?" "\\(db-based-graph\\?"
                          ;; Use file-entity-util and entity-util when in a single graph context
                          "ldb/whiteboard\\?" "ldb/journal\\?" "ldb/page\\?"]
         res (grep-many multi-graph-fns (into file-graph-paths db-graph-paths))]
@@ -120,12 +120,16 @@
                               "block/properties :"
                               ;; anything org mode except for org.babashka or urls like schema.org
                               "[^\\.]org[^\\.]"
+                              "file-based"
                               "#+BEGIN_"
                               "#+END_"
                               "pre-block"]))
         ;; For now use the whole code line. If this is too brittle can make this smaller
         allowed-exceptions #{":block/pre-block? :block/scheduled :block/deadline :block/type :block/name :block/marker"
                              "(dissoc :block/format))]"
+                             ;; TODO: Mv these 2 file-based ns out of db files
+                             "(:require [logseq.db.file-based.rules :as file-rules]))"
+                             "[logseq.db.file-based.schema :as file-schema]))"
                              ;; The next 3 are from components.property.value
                              "{:block/name page-title})"
                              "(when-not (db/get-page journal)"
@@ -159,7 +163,7 @@
   (let [db-concepts
         ;; from logseq.db.frontend.schema
         ["closed-value" "class/properties" "classes" "property/parent"
-         "logseq.property" "logseq.class"]
+         "logseq.property" "logseq.class" "db-based"]
         res (grep-many db-concepts file-graph-paths)]
     (when-not (and (= 1 (:exit res)) (= "" (:out res)))
       (println "The following files should not have contained db specific concepts:")

src/electron/electron/db.cljs

@@ -3,7 +3,7 @@
   (:require ["electron" :refer [app]]
             ["fs-extra" :as fs]
             ["path" :as node-path]
-            [logseq.db.common.sqlite :as sqlite-common-db]))
+            [logseq.db.common.sqlite :as common-sqlite]))
 
 (defn get-graphs-dir
   []
@@ -17,19 +17,19 @@
 (defn ensure-graph-dir!
   [db-name]
   (ensure-graphs-dir!)
-  (let [graph-dir (node-path/join (get-graphs-dir) (sqlite-common-db/sanitize-db-name db-name))]
+  (let [graph-dir (node-path/join (get-graphs-dir) (common-sqlite/sanitize-db-name db-name))]
     (fs/ensureDirSync graph-dir)
     graph-dir))
 
 (defn save-db!
   [db-name data]
-  (let [[_db-name db-path] (sqlite-common-db/get-db-full-path (get-graphs-dir) db-name)]
+  (let [[_db-name db-path] (common-sqlite/get-db-full-path (get-graphs-dir) db-name)]
     (fs/writeFileSync db-path data)))
 
 (defn get-db
   [db-name]
   (let [_ (ensure-graph-dir! db-name)
-        [_db-name db-path] (sqlite-common-db/get-db-full-path (get-graphs-dir) db-name)]
+        [_db-name db-path] (common-sqlite/get-db-full-path (get-graphs-dir) db-name)]
     (when (fs/existsSync db-path)
       (fs/readFileSync db-path))))
 
@@ -37,7 +37,7 @@
 
 (defn unlink-graph!
   [repo]
-  (let [db-name (sqlite-common-db/sanitize-db-name repo)
+  (let [db-name (common-sqlite/sanitize-db-name repo)
         path (node-path/join (get-graphs-dir) db-name)
         unlinked (node-path/join (get-graphs-dir) unlinked-graphs-dir)
         new-path (node-path/join unlinked db-name)

src/electron/electron/handler.cljs

@@ -31,7 +31,7 @@
             [electron.window :as win]
             [goog.functions :refer [debounce]]
             [logseq.common.graph :as common-graph]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.sqlite.util :as sqlite-util]
             [promesa.core :as p]))
 
@@ -240,7 +240,7 @@
          (remove (fn [s] (= s db/unlinked-graphs-dir)))
          (map graph-name->path)
          (map (fn [s]
-                (if (string/starts-with? s sqlite-util/file-version-prefix)
+                (if (string/starts-with? s common-sqlite/file-version-prefix)
                   s
                   (str sqlite-util/db-version-prefix s)))))))
 
@@ -294,7 +294,7 @@
 (defmethod handle :deleteGraph [_window [_ graph graph-name _db-based?]]
   (when graph-name
     (db/unlink-graph! graph)
-    (let [old-transit-path (node-path/join (get-graphs-dir) (str (sqlite-common-db/sanitize-db-name graph) ".transit"))]
+    (let [old-transit-path (node-path/join (get-graphs-dir) (str (common-sqlite/sanitize-db-name graph) ".transit"))]
       (when (fs/existsSync old-transit-path)
         (fs/unlinkSync old-transit-path)))))
 

src/main/frontend/background_tasks.cljs

@@ -2,11 +2,11 @@
   "Some background tasks"
   (:require [frontend.common.missionary :as c.m]
             [frontend.flows :as flows]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [missionary.core :as m]))
 
 (c.m/run-background-task
- :logseq.db.frontend.entity-plus/reset-immutable-entities-cache!
+ :logseq.db.common.entity-plus/reset-immutable-entities-cache!
  (m/reduce
   (fn [_ repo]
     (when (some? repo)

src/main/frontend/commands.cljs

@@ -357,10 +357,20 @@
 
       ;; task management
       (get-statuses)
-      [["Deadline" [[:editor/clear-current-slash]
-                    [:editor/set-deadline]] "" :icon/calendar-stats]
-       ["Scheduled" [[:editor/clear-current-slash]
-                     [:editor/set-scheduled]] "" :icon/calendar-month]]
+
+      ;; task date
+      [["Deadline"
+        [[:editor/clear-current-slash]
+         [:editor/set-deadline]]
+        ""
+        :icon/calendar-stats
+        "TASK DATE"]
+       ["Scheduled"
+        [[:editor/clear-current-slash]
+         [:editor/set-scheduled]]
+        ""
+        :icon/calendar-month
+        "TASK DATE"]]
 
       ;; priority
       (get-priorities)

src/main/frontend/common/file/core.cljs

@@ -5,7 +5,7 @@
             [datascript.core :as d]
             [logseq.db :as ldb]
             [logseq.db.frontend.content :as db-content]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.property :as gp-property]
             [logseq.outliner.tree :as otree]))

src/main/frontend/common/graph_view.cljs

@@ -6,7 +6,7 @@
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db.common.property-util :as db-property-util]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.graph-parser.db :as gp-db]))
 

src/main/frontend/components/block.cljs

@@ -80,7 +80,7 @@
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
             [logseq.db.frontend.content :as db-content]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.text :as text]

src/main/frontend/components/cmdk/core.cljs

@@ -197,16 +197,17 @@
     "page"))
 
 (defmethod load-results :initial [_ state]
-  (let [!results (::results state)
-        recent-pages (map (fn [block]
-                            (let [text (block-handler/block-unique-title block)
-                                  icon (get-page-icon block)]
-                              {:icon icon
-                               :icon-theme :gray
-                               :text text
-                               :source-block block}))
-                          (ldb/get-recent-updated-pages (db/get-db)))]
-    (reset! !results (assoc-in default-results [:recently-updated-pages :items] recent-pages))))
+  (when-let [db (db/get-db)]
+    (let [!results (::results state)
+          recent-pages (map (fn [block]
+                              (let [text (block-handler/block-unique-title block)
+                                    icon (get-page-icon block)]
+                                {:icon icon
+                                 :icon-theme :gray
+                                 :text text
+                                 :source-block block}))
+                            (ldb/get-recent-updated-pages db))]
+      (reset! !results (assoc-in default-results [:recently-updated-pages :items] recent-pages)))))
 
 ;; The commands search uses the command-palette handler
 (defmethod load-results :commands [group state]

src/main/frontend/components/editor.cljs

@@ -47,7 +47,7 @@
               (or
                (= "Add new property" (first item))
                (when (= (count item) 5)
-                 (contains? #{"TASK STATUS" "PRIORITY"} (last item))))) commands)
+                 (contains? #{"TASK STATUS" "TASK DATE" "PRIORITY"} (last item))))) commands)
     commands))
 
 (rum/defcs commands < rum/reactive

src/main/frontend/components/server.cljs

@@ -74,7 +74,7 @@
                          (not= (util/safe-parse-int (or port 0))
                                (util/safe-parse-int (or (:port server-state) 0))))
         changed?     (or hp-changed? (->> [autostart (:autostart server-state)]
-                                          (mapv #(cond-> % (nil? %) not))
+                                          (mapv #(cond-> % (nil? %) boolean))
                                           (apply not=)))]
 
     [:div.cp__server-configs-panel.pt-5
@@ -104,7 +104,7 @@
        (ui/checkbox
         {:on-change #(let [checked (.-checked (.-target %))]
                        (swap! *configs assoc :autostart checked))
-         :value     (not (false? autostart))})
+         :checked   (not (false? autostart))})
 
        [:strong.select-none "Auto start server with the app launched"]]]
 

src/main/frontend/db/restore.cljs

@@ -1,12 +1,12 @@
 (ns frontend.db.restore
   "Fns for DB restore(from text or sqlite)"
   (:require [cljs-time.core :as t]
+            [datascript.core :as d]
             [frontend.db.conn :as db-conn]
             [frontend.persist-db :as persist-db]
             [frontend.state :as state]
             [frontend.undo-redo :as undo-redo]
             [logseq.db :as ldb]
-            [logseq.db.common.sqlite :as sqlite-common-db]
             [promesa.core :as p]))
 
 (defn restore-graph!
@@ -15,11 +15,12 @@
   (state/set-state! :graph/loading? true)
   (p/let [start-time (t/now)
           {:keys [schema initial-data]} (persist-db/<fetch-init-data repo opts)
+          _ (state/set-current-repo! repo)
           ;; Without valid schema app fails hard downstream
           _ (when (nil? schema)
               (throw (ex-info "No valid schema found when reloading db" {:repo repo})))
           conn (try
-                 (sqlite-common-db/restore-initial-data initial-data schema)
+                 (d/conn-from-datoms initial-data schema)
                  (catch :default e
                    (prn :error :restore-initial-data-failed
                         (ldb/write-transit-str {:schema schema

src/main/frontend/db/utils.cljs

@@ -4,7 +4,7 @@
             [frontend.db.conn :as conn]
             [frontend.state :as state]
             [logseq.db.frontend.content :as db-content]
-            [logseq.db.frontend.entity-plus :as entity-plus]))
+            [logseq.db.common.entity-plus :as entity-plus]))
 
 ;; transit serialization
 

src/main/frontend/extensions/fsrs.cljs

@@ -18,7 +18,7 @@
             [frontend.ui :as ui]
             [frontend.util :as util]
             [logseq.db :as ldb]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.shui.ui :as shui]
             [missionary.core :as m]
             [open-spaced-repetition.cljc-fsrs.core :as fsrs.core]

src/main/frontend/handler/db_based/rtc.cljs

@@ -10,7 +10,7 @@
             [frontend.util :as util]
             [lambdaisland.glogi :as log]
             [logseq.db :as ldb]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.shui.ui :as shui]
             [promesa.core :as p]))
 
@@ -19,7 +19,7 @@
   (p/do!
    (js/Promise. user-handler/task--ensure-id&access-token)
    (let [token (state/get-auth-id-token)
-         repo-name (sqlite-common-db/sanitize-db-name repo)]
+         repo-name (common-sqlite/sanitize-db-name repo)]
      (state/<invoke-db-worker :thread-api/rtc-async-upload-graph repo token repo-name))))
 
 (defn <rtc-delete-graph!

src/main/frontend/handler/editor.cljs

@@ -55,7 +55,7 @@
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
             [logseq.db.file-based.schema :as file-schema]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]

src/main/frontend/handler/export.cljs

@@ -22,7 +22,7 @@
    [goog.dom :as gdom]
    [lambdaisland.glogi :as log]
    [logseq.db :as ldb]
-   [logseq.db.common.sqlite :as sqlite-common-db]
+   [logseq.db.common.sqlite :as common-sqlite]
    [logseq.publishing.html :as publish-html]
    [promesa.core :as p])
   (:import
@@ -58,7 +58,7 @@
   [repo]
   (p/let [db-data (persist-db/<export-db repo {:return-data? true})
           filename "db.sqlite"
-          repo-name (sqlite-common-db/sanitize-db-name repo)
+          repo-name (common-sqlite/sanitize-db-name repo)
           assets (assets-handler/<get-all-assets)
           files (cons [filename db-data] assets)
           zipfile (zip/make-zip repo-name files repo)]
@@ -255,7 +255,7 @@
   (when (and repo (= repo (state/get-current-repo)))
     (when-let [backup-folder (ldb/get-key-value (db/get-db repo) :logseq.kv/graph-backup-folder)]
       (p/let [handle (idb/get-item (str "handle/" (js/btoa repo) "/" backup-folder))
-              repo-name (sqlite-common-db/sanitize-db-name repo)]
+              repo-name (common-sqlite/sanitize-db-name repo)]
         (if handle
           (->
            (p/let [graph-dir-handle (.getDirectoryHandle handle repo-name #js {:create true})

src/main/frontend/state.cljs

@@ -21,7 +21,7 @@
             [goog.object :as gobj]
             [logseq.common.config :as common-config]
             [logseq.db :as ldb]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.shui.dialog.core :as shui-dialog]
             [logseq.shui.hooks :as hooks]

src/main/frontend/worker/db/fix.cljs

@@ -1,11 +1,11 @@
 (ns frontend.worker.db.fix
   "fix db"
   (:require [datascript.core :as d]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db :as ldb]))
 
 (defn check-and-fix-schema!
   [repo conn]
-  (let [schema (sqlite-util/get-schema repo)
+  (let [schema (ldb/get-schema repo)
         db-schema (:schema @conn)
         diffs (->> (keep (fn [[k v]]
                            (let [schema-v (-> (get db-schema k)

src/main/frontend/worker/db_worker.cljs

@@ -31,12 +31,14 @@
             [frontend.worker.util :as worker-util]
             [goog.object :as gobj]
             [lambdaisland.glogi.console :as glogi-console]
+            [logseq.common.log :as log]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
+            [logseq.db.common.entity-plus :as entity-plus]
+            [logseq.db.common.initial-data :as common-initial-data]
             [logseq.db.common.order :as db-order]
-            [logseq.db.common.sqlite :as sqlite-common-db]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.common.view :as db-view]
-            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.export :as sqlite-export]
@@ -328,19 +330,19 @@
                                        :client-ops client-ops-db})
       (doseq [db' dbs]
         (enable-sqlite-wal-mode! db'))
-      (sqlite-common-db/create-kvs-table! db)
-      (when-not @*publishing? (sqlite-common-db/create-kvs-table! client-ops-db))
+      (common-sqlite/create-kvs-table! db)
+      (when-not @*publishing? (common-sqlite/create-kvs-table! client-ops-db))
       (db-migrate/migrate-sqlite-db db)
       (when-not @*publishing? (db-migrate/migrate-sqlite-db client-ops-db))
       (search/create-tables-and-triggers! search-db)
-      (let [schema (sqlite-util/get-schema repo)
-            conn (sqlite-common-db/get-storage-conn storage schema)
+      (let [schema (ldb/get-schema repo)
+            conn (common-sqlite/get-storage-conn storage schema)
             _ (db-fix/check-and-fix-schema! repo conn)
             _ (when datoms
                 (let [data (map (fn [datom]
                                   [:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
                   (d/transact! conn data {:initial-db? true})))
-            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn
+            client-ops-conn (when-not @*publishing? (common-sqlite/get-storage-conn
                                                      client-ops-storage
                                                      client-op/schema-in-db))
             initial-data-exists? (when (nil? datoms)
@@ -357,12 +359,14 @@
             (d/transact! conn initial-data {:initial-db? true})))
 
         (try
-          (when-let [missing-addresses (seq (find-missing-addresses db))]
-            (throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses})))
+          (when-not import-type
+            (when-let [missing-addresses (seq (find-missing-addresses db))]
+              (throw (ex-info "DB missing addresses" {:missing-addresses missing-addresses}))))
 
           (db-migrate/migrate conn search-db)
 
           (catch :default _e
+            (log/error "DB migrate failed, retrying")
             (when db-based?
               (rebuild-db-from-datoms! conn db import-type)
               (db-migrate/migrate conn search-db))))
@@ -489,7 +493,7 @@
                 id)]
       (some->> eid
                (d/pull @conn selector)
-               (sqlite-common-db/with-parent @conn)))))
+               (common-initial-data/with-parent @conn)))))
 
 (def ^:private *get-blocks-cache (volatile! (cache/lru-cache-factory {} :threshold 1000)))
 (def ^:private get-blocks-with-cache
@@ -503,7 +507,7 @@
      (when db
        (mapv (fn [{:keys [id opts]}]
                (let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)]
-                 (-> (sqlite-common-db/get-block-and-children db id' opts)
+                 (-> (common-initial-data/get-block-and-children db id' opts)
                      (assoc :id id)))) requests)))))
 
 (def-thread-api :thread-api/get-blocks
@@ -601,7 +605,7 @@
 (def-thread-api :thread-api/get-initial-data
   [repo]
   (when-let [conn (worker-state/get-datascript-conn repo)]
-    (sqlite-common-db/get-initial-data @conn)))
+    (common-initial-data/get-initial-data @conn)))
 
 (def-thread-api :thread-api/reset-db
   [repo db-transit]
@@ -851,7 +855,7 @@
          :rtc-sync-state])))
 
 (defn- <init-service!
-  [graph]
+  [graph import?]
   (let [[prev-graph service] @*service]
     (some-> prev-graph close-db!)
     (when graph
@@ -860,7 +864,8 @@
         (p/let [service (shared-service/<create-service graph
                                                         (bean/->js fns)
                                                         #(on-become-master graph)
-                                                        broadcast-data-types)]
+                                                        broadcast-data-types
+                                                        {:import? import?})]
           (assert (p/promise? (get-in service [:status :ready])))
           (reset! *service [graph service])
           service)))))
@@ -880,10 +885,10 @@
                                 (= :thread-api/create-or-open-db method-k)
                                 ;; because shared-service operates at the graph level,
                                 ;; creating a new database or switching to another one requires re-initializing the service.
-                                (let [[graph _opts] (ldb/read-transit-str (last args))]
-                                  (p/let [service (<init-service! graph)]
+                                (let [[graph opts] (ldb/read-transit-str (last args))]
+                                  (p/let [service (<init-service! graph (some? (:import-type opts)))]
                                     (get-in service [:status :ready])
-                                      ;; wait for service ready
+                                    ;; wait for service ready
                                     (js-invoke (:proxy service) k args)))
 
                                 (or (contains? #{:thread-api/sync-app-state} method-k)

src/main/frontend/worker/handler/page/db_based/page.cljs

@@ -9,7 +9,7 @@
             [logseq.db.common.order :as db-order]
             [logseq.db.common.property-util :as db-property-util]
             [logseq.db.frontend.class :as db-class]
-            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.common.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.property.build :as db-property-build]

src/main/frontend/worker/pipeline.cljs

@@ -12,6 +12,7 @@
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
+            [logseq.db.common.sqlite :as common-sqlite]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.sqlite.util :as sqlite-util]
@@ -232,7 +233,7 @@
                                 :db-after (:db-after result)))
                        tx-report)
           {:keys [pages blocks]} (ds-report/get-blocks-and-pages tx-report*)
-          _ (when (sqlite-util/local-file-based-graph? repo)
+          _ (when (common-sqlite/local-file-based-graph? repo)
               (let [page-ids (distinct (map :db/id pages))]
                 (doseq [page-id page-ids]
                   (when (d/entity @conn page-id)

src/main/frontend/worker/shared_service.cljs

@@ -302,8 +302,9 @@
 (defn <create-service
   "broadcast-data-types - For data matching these types,
                           forward the data broadcast from the master client directly to the UI thread."
-  [service-name target on-become-master-handler broadcast-data-types]
+  [service-name target on-become-master-handler broadcast-data-types {:keys [import?]}]
   (clear-old-service!)
+  (when import? (reset! *master-client? true))
   (p/let [broadcast-data-types (set broadcast-data-types)
           status {:ready (p/deferred)}
           common-channel (ensure-common-channel service-name)

src/main/frontend/worker/util.cljc

@@ -6,7 +6,7 @@
                      [goog.crypt :as crypt]
                      [goog.crypt.Hmac]
                      [goog.crypt.Sha256]
-                     [logseq.db.common.sqlite :as sqlite-common-db]
+                     [logseq.db.common.sqlite :as common-sqlite]
                      [frontend.common.file.util :as wfu])))
 
 ;; Copied from https://github.com/tonsky/datascript-todo
@@ -27,7 +27,7 @@
 
      (defn get-pool-name
        [graph-name]
-       (str "logseq-pool-" (sqlite-common-db/sanitize-db-name graph-name)))
+       (str "logseq-pool-" (common-sqlite/sanitize-db-name graph-name)))
 
      (defn- decode-username
        [username]