Merge branch 'feat/db' into perf/app-start

.clj-kondo/config.edn

@@ -102,6 +102,7 @@
              frontend.handler.common.plugin plugin-common-handler
              frontend.handler.config config-handler
              frontend.handler.db-based.editor db-editor-handler
+             frontend.handler.db-based.export db-export-handler
              frontend.handler.db-based.page db-page-handler
              frontend.handler.db-based.property db-property-handler
              frontend.handler.db-based.property.util db-pu
@@ -216,7 +217,8 @@
                         rum.core/defcs hooks.rum/defcs
                         clojure.string/join hooks.path-invalid-construct/string-join
                         clojure.string/replace hooks.regex-checks/double-escaped-regex
-                        logseq.common.defkeywords/defkeywords hooks.defkeywords/defkeywords}}
+                        logseq.common.defkeywords/defkeywords hooks.defkeywords/defkeywords
+                        frontend.common.thread-api/def-thread-api hooks.def-thread-api/def-thread-api}}
  :lint-as {promesa.core/let clojure.core/let
            promesa.core/loop clojure.core/loop
            promesa.core/recur clojure.core/recur
@@ -232,7 +234,8 @@
            frontend.test.helper/deftest-async clojure.test/deftest
            frontend.worker.rtc.idb-keyval-mock/with-reset-idb-keyval-mock cljs.test/async
            frontend.react/defc clojure.core/defn
-           logseq.common.defkeywords/defkeyword cljs.spec.alpha/def}
+           logseq.common.defkeywords/defkeyword cljs.spec.alpha/def
+           frontend.common.thread-api/defkeyword cljs.spec.alpha/def}
  :skip-comments true
  :output {:progress true
           :exclude-files ["src/test/docs-0.10.9/"]}}

.clj-kondo/hooks/def_thread_api.clj

@@ -0,0 +1,13 @@
+(ns hooks.def-thread-api
+  (:require [clj-kondo.hooks-api :as api]))
+
+(defn def-thread-api
+  [{:keys [node]}]
+  (let [[_ kw & others] (:children node)
+        new-node (api/list-node
+                  [(api/token-node 'do)
+                   (api/list-node [(api/token-node 'frontend.common.thread-api/defkeyword) kw])
+                   (api/list-node
+                    (cons (api/token-node 'fn) others))])
+        new-node* (with-meta new-node (meta node))]
+    {:node new-node*}))

.github/workflows/build.yml

@@ -165,17 +165,23 @@ jobs:
         run: cd scripts && yarn install --frozen-lockfile
 
       - name: Create DB graph with properties
-        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs ./db-graph-with-props
+        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs ./properties-graph
 
       # TODO: Use a smaller, test-focused graph to test classes
       - name: Create DB graph with classes
-        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs ./db-graph-with-schema
+        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs ./schema-graph
 
       - name: Fetch deps/db yarn deps
         run: cd deps/db && yarn install --frozen-lockfile
 
       - name: Validate created DB graphs
-        run: cd deps/db && yarn nbb-logseq script/validate_db.cljs ../../scripts/db-graph-with-props ../../scripts/db-graph-with-schema --closed-maps --group-errors
+        run: cd deps/db && yarn nbb-logseq script/validate_db.cljs ../../scripts/properties-graph ../../scripts/schema-graph --closed-maps --group-errors
+
+      - name: Export a created DB graph
+        run: cd deps/db && yarn nbb-logseq script/export_graph.cljs ../../scripts/properties-graph -f properties.edn -t
+
+      - name: Create graph from the export and diff the two graphs
+        run: cd deps/db && yarn nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs ./properties-graph2 properties.edn -iv && yarn nbb-logseq script/diff_graphs.cljs ../../scripts/properties-graph ./properties-graph2 -t
 
   e2e-test:
     # TODO: Re-enable when ready to enable tests for file graphs

bb.edn

@@ -75,7 +75,9 @@
 
   dev:db-transact
   {:doc "Transact against a DB graph's datascript db"
-   :task (apply shell {:dir "deps/outliner"} "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/outliner" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq script/transact.cljs" *command-line-args*)}
 
   dev:db-create
   {:doc "Create a DB graph given a sqlite.build EDN file"
@@ -83,6 +85,18 @@
    :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
                 "yarn -s nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs" *command-line-args*)}
 
+  dev:db-export
+  {:doc "Export a DB graph to a sqlite.build EDN file"
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq script/export_graph.cljs" *command-line-args*)}
+
+  dev:db-diff
+  {:doc "Diffs two DB graphs"
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq script/diff_graphs.cljs" *command-line-args*)}
+
   dev:db-import
   {:doc "Import a file graph to db graph"
    :requires ([babashka.fs :as fs])

deps/common/src/logseq/common/defkeywords.cljc

@@ -33,3 +33,8 @@
 (defmacro get-all-defined-kw->config
   []
   `'~(deref *defined-kw->config))
+
+(comment
+  "update anything here to trigger this ns to be recompiled,
+so macro get-all-defined-kw->config's result will be updated."
+  1)

deps/common/src/logseq/common/uuid.cljs

@@ -12,7 +12,7 @@ the remaining chars for data of this type"
   (let [journal-day-str  (str journal-day)
         part1 (subs journal-day-str 0 4)
         part2 (subs journal-day-str 4 8)]
-    (uuid (str "00000001-" part1 "-" part2 "-0000-000000000000"))))
+    (uuid (str "00000001" "-" part1 "-" part2 "-0000-000000000000"))))
 
 (defn- fill-with-0
   [s length]
@@ -49,3 +49,16 @@ the remaining chars for data of this type"
      :db-ident-block-uuid (gen-db-ident-block-uuid v)
      :migrate-new-block-uuid (gen-block-uuid v "00000003")
      :builtin-block-uuid (gen-block-uuid v "00000004"))))
+
+(defn gen-journal-template-block
+  "Persistent uuid for journal template block"
+  [journal-uuid template-block-uuid]
+  (assert (uuid? journal-uuid) (str journal-uuid))
+  (assert (uuid? template-block-uuid) (str template-block-uuid))
+  (uuid
+   (str "00000005"
+        "-"
+       ;; journal day
+        (subs (str journal-uuid) 9 23)
+       ;; template block uuid
+        (subs (str template-block-uuid) 23))))

deps/db/script/create_graph.cljs

@@ -1,5 +1,6 @@
 (ns create-graph
-  "A script that creates a DB graph given a sqlite.build EDN file"
+  "A script that creates or updates a DB graph given a sqlite.build EDN file.
+   If the given graph already exists, the EDN file updates the graph."
   (:require ["fs" :as fs]
             ["os" :as os]
             ["path" :as node-path]
@@ -8,6 +9,7 @@
             [clojure.edn :as edn]
             [clojure.string :as string]
             [datascript.core :as d]
+            [logseq.db.sqlite.export :as sqlite-export]
             [logseq.outliner.cli :as outliner-cli]
             [nbb.classpath :as cp]
             [nbb.core :as nbb]
@@ -20,12 +22,25 @@
     path
     (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (def spec
   "Options spec"
   {:help {:alias :h
           :desc "Print help"}
    :validate {:alias :v
-              :desc "Validate db after creation"}})
+              :desc "Validate db after creation"}
+   :import {:alias :i
+            :desc "Import edn file using sqlite-export"}})
 
 (defn -main [args]
   (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
@@ -34,19 +49,23 @@
             (println (str "Usage: $0 GRAPH-NAME EDN-PATH [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        [dir db-name] (if (string/includes? graph-dir "/")
-                        ((juxt node-path/dirname node-path/basename) graph-dir)
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
-        sqlite-build-edn (merge {:auto-create-ontology? true}
+        [dir db-name] (get-dir-and-db-name graph-dir)
+        sqlite-build-edn (merge (if (:import options) {} {:auto-create-ontology? true})
                                 (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
+        graph-exists? (fs/existsSync (node-path/join dir db-name))
         conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph})
-        {:keys [init-tx block-props-tx] :as _txs} (outliner-cli/build-blocks-tx sqlite-build-edn)]
+        {:keys [init-tx block-props-tx misc-tx] :as _txs}
+        (if (:import options)
+          (sqlite-export/build-import sqlite-build-edn @conn {})
+          (outliner-cli/build-blocks-tx sqlite-build-edn))]
     (println "Generating" (count (filter :block/name init-tx)) "pages and"
              (count (filter :block/title init-tx)) "blocks ...")
+    ;; (fs/writeFileSync "txs.edn" (with-out-str (cljs.pprint/pprint _txs)))
     ;; (cljs.pprint/pprint _txs)
     (d/transact! conn init-tx)
-    (d/transact! conn block-props-tx)
-    (println "Created graph" (str db-name "!"))
+    (when (seq block-props-tx) (d/transact! conn block-props-tx))
+    (when (seq misc-tx) (d/transact! conn misc-tx))
+    (println (if graph-exists? "Updated graph" "Created graph") (str db-name "!"))
     (when (:validate options)
       (validate-db/validate-db @conn db-name {:group-errors true :closed-maps true :humanize true}))))
 

deps/db/script/diff_graphs.cljs

@@ -0,0 +1,75 @@
+(ns diff-graphs
+  "A script that diffs two DB graphs through their sqlite.build EDN"
+  (:require ["os" :as os]
+            ["path" :as node-path]
+            [babashka.cli :as cli]
+            [clojure.data :as data]
+            [clojure.pprint :as pprint]
+            [clojure.string :as string]
+            [logseq.common.config :as common-config]
+            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [nbb.core :as nbb]))
+
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
+(def spec
+  "Options spec"
+  {:help {:alias :h
+          :desc "Print help"}
+   :exclude-namespaces {:alias :e
+                        :coerce #{}
+                        :desc "Namespaces to exclude from properties and classes"}
+   :exclude-built-in-pages? {:alias :b
+                             :desc "Exclude built-in pages"}
+   :set-diff {:alias :s
+              :desc "Use set to reduce noisy diff caused by ordering"}
+   :include-timestamps? {:alias :t
+                         :desc "Include timestamps in export"}})
+
+(defn -main [args]
+  (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
+        [graph-dir graph-dir2] args'
+        _ (when (or (nil? graph-dir) (nil? graph-dir2) (:help options))
+            (println (str "Usage: $0 GRAPH-NAME GRAPH-NAME2 [& ARGS] [OPTIONS]\nOptions:\n"
+                          (cli/format-opts {:spec spec})))
+            (js/process.exit 1))
+        conn (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir))
+        conn2 (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir2))
+        export-options (select-keys options [:include-timestamps? :exclude-namespaces :exclude-built-in-pages?])
+        export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})
+        export-map2 (sqlite-export/build-export @conn2 {:export-type :graph :graph-options export-options})
+        prepare-export-to-diff
+        (fn [m]
+          (cond->
+           (-> m
+               ;; TODO: Fix order of these build keys
+               (update :classes update-vals (fn [m] (update m :build/class-properties sort)))
+               (update :properties update-vals (fn [m] (update m :build/property-classes sort)))
+               (update ::sqlite-export/kv-values
+                       (fn [kvs]
+                         ;; Ignore extra metadata that a copied graph can add
+                         (vec (remove #(#{:logseq.kv/import-type :logseq.kv/imported-at} (:db/ident %)) kvs))))
+              ;; TODO: fix built-in views for schema export
+               (update :pages-and-blocks (fn [pbs]
+                                           (vec (remove #(= (:block/title (:page %)) common-config/views-page-name) pbs)))))
+            (:set-diff options)
+            (update-vals set)))
+        diff (->> (data/diff (prepare-export-to-diff export-map) (prepare-export-to-diff export-map2))
+                  butlast)]
+    (if (= diff [nil nil])
+      (println "The two graphs are equal!")
+      (do (pprint/pprint diff)
+          (js/process.exit 1)))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/script/export_graph.cljs

@@ -0,0 +1,71 @@
+(ns export-graph
+  "A script that exports a graph to a sqlite.build EDN file"
+  (:require ["fs" :as fs]
+            ["os" :as os]
+            ["path" :as node-path]
+            [babashka.cli :as cli]
+            [clojure.edn :as edn]
+            [clojure.pprint :as pprint]
+            [clojure.string :as string]
+            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [nbb.core :as nbb]))
+
+(defn- resolve-path
+  "If relative path, resolve with $ORIGINAL_PWD"
+  [path]
+  (if (node-path/isAbsolute path)
+    path
+    (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
+
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
+(def spec
+  "Options spec"
+  {:help {:alias :h
+          :desc "Print help"}
+   :include-timestamps? {:alias :t
+                         :desc "Include timestamps in export"}
+   :file {:alias :f
+          :desc "Saves edn to file"}
+   :catch-validation-errors? {:alias :c
+                              :desc "Catch validation errors for dev"}
+   :exclude-namespaces {:alias :e
+                        :coerce #{}
+                        :desc "Namespaces to exclude from properties and classes"}
+   :exclude-built-in-pages? {:alias :b
+                             :desc "Exclude built-in pages"}
+   :exclude-files? {:alias :F
+                    :desc "Exclude :file/path files"}})
+
+(defn -main [args]
+  (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
+        graph-dir (first args')
+        _ (when (or (nil? graph-dir) (:help options))
+            (println (str "Usage: $0 GRAPH-NAME [& ARGS] [OPTIONS]\nOptions:\n"
+                          (cli/format-opts {:spec spec})))
+            (js/process.exit 1))
+        [dir db-name] (get-dir-and-db-name graph-dir)
+        conn (sqlite-cli/open-db! dir db-name)
+        export-options (dissoc options :file)
+        export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})]
+    (if (:file options)
+      (do
+        (println "Exported" (count (:properties export-map)) "properties,"
+                 (count (:properties export-map)) "classes and"
+                 (count (:pages-and-blocks export-map)) "pages")
+        (fs/writeFileSync (resolve-path (:file options))
+                          (with-out-str (pprint/pprint export-map))))
+      (pprint/pprint export-map))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/script/query.cljs

@@ -23,12 +23,14 @@
                            (clj->js (merge {:stdio "inherit"} opts))))
 
 (defn- get-dir-and-db-name
-  "Gets dir and db name for use with open-db!"
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
   [graph-dir]
   (if (string/includes? graph-dir "/")
-    (let [graph-dir'
-          (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
-      ((juxt node-path/dirname node-path/basename) graph-dir'))
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
     [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
 
 (def spec

deps/db/script/validate_db.cljs

@@ -50,6 +50,17 @@
         (js/process.exit 1))
       (println "Valid!"))))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -75,11 +86,7 @@
     (validate-db* db ent-maps options)))
 
 (defn- validate-graph [graph-dir options]
-  (let [[dir db-name] (if (string/includes? graph-dir "/")
-                        (let [graph-dir'
-                              (node-path/join (or js/process.env.ORIGINAL_PWD ".") graph-dir)]
-                          ((juxt node-path/dirname node-path/basename) graph-dir'))
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
+  (let [[dir db-name] (get-dir-and-db-name graph-dir)
         conn (try (sqlite-cli/open-db! dir db-name)
                   (catch :default e
                     (println "Error: For graph" (str (pr-str graph-dir) ":") (str e))

deps/db/src/logseq/db/frontend/db_ident.cljc

@@ -68,7 +68,7 @@
          :cljs (exists? js/process)
          :default false)
     ;; So that we don't have to change :user.{property|class} in our tests
-    (keyword user-namespace (-> name-string (string/replace #"/|\s+" "-") (string/replace-first #"^(\d)" "NUM-$1")))
+    (keyword user-namespace (-> name-string (string/replace #"[/()]|\s+" "-") (string/replace-first #"^(\d)" "NUM-$1")))
     (keyword user-namespace
              (str
               (->> (filter #(re-find #"[0-9a-zA-Z-]{1}" %) (seq name-string)) (apply str))

deps/db/src/logseq/db/frontend/property.cljs

@@ -407,10 +407,7 @@
              [:logseq.property.view/type.list "List View" "list"]
              [:logseq.property.view/type.gallery "Gallery View" "layout-grid"]])
       :properties {:logseq.property/default-value :logseq.property.view/type.table}
-      :queryable? true
-      :rtc {:rtc/ignore-attr-when-init-upload true
-            :rtc/ignore-attr-when-init-download true
-            :rtc/ignore-attr-when-syncing true}}
+      :queryable? true}
 
      :logseq.property.view/feature-type
      {:title "View Feature Type"
@@ -418,10 +415,7 @@
       {:type :keyword
        :public? false
        :hide? true}
-      :queryable? false
-      :rtc {:rtc/ignore-attr-when-init-upload true
-            :rtc/ignore-attr-when-init-download true
-            :rtc/ignore-attr-when-syncing true}}
+      :queryable? false}
 
      :logseq.property.view/group-by-property
      {:title "View group by property"
@@ -429,10 +423,7 @@
       {:type :property
        :public? false
        :hide? true}
-      :queryable? true
-      :rtc {:rtc/ignore-attr-when-init-upload true
-            :rtc/ignore-attr-when-init-download true
-            :rtc/ignore-attr-when-syncing true}}
+      :queryable? true}
 
      :logseq.property.table/sorting {:title "View sorting"
                                      :schema
@@ -664,6 +655,15 @@
   [s]
   (string/includes? s ".class"))
 
+(defn internal-property?
+  "Determines if ident kw is an internal property. This includes db-attribute properties
+   unlike logseq-property? and doesn't include non-property idents unlike internal-ident?"
+  [k]
+  (let [k-name (namespace k)]
+    (and k-name
+         (or (contains? logseq-property-namespaces k-name)
+             (contains? public-db-attribute-properties k)))))
+
 (defn property?
   "Determines if ident kw is a property visible to user"
   [k]

deps/db/src/logseq/db/sqlite/build.cljs

@@ -8,6 +8,7 @@
   (:require [cljs.pprint :as pprint]
             [clojure.set :as set]
             [clojure.string :as string]
+            [clojure.walk :as walk]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
@@ -46,6 +47,8 @@
           [:block/uuid page-uuid]
           (throw (ex-info (str "No uuid for page '" (second val) "'") {:name (second val)}))))
       :block/uuid
+      val
+      ;; Allow through :coll properties like
       val)
     val))
 
@@ -68,17 +71,19 @@
     (or (get all-idents kw)
         (throw (ex-info (str "No ident found for " (pr-str kw)) {})))))
 
-(defn- ->block-properties [properties page-uuids all-idents]
-  (->>
-   (map
-    (fn [[prop-name val]]
-      [(get-ident all-idents prop-name)
-       ;; set indicates a :many value
-       (if (set? val)
-         (set (map #(translate-property-value % page-uuids) val))
-         (translate-property-value val page-uuids))])
-    properties)
-   (into {})))
+(defn- ->block-properties [properties page-uuids all-idents {:keys [translate-property-values?]}]
+  (let [translate-property-values (if translate-property-values?
+                                    (fn translate-property-values [val]
+                                      ;; set indicates a :many value
+                                      (if (set? val)
+                                        (set (map #(translate-property-value % page-uuids) val))
+                                        (translate-property-value val page-uuids)))
+                                    identity)]
+    (->> (map (fn [[prop-name val]]
+                [(get-ident all-idents prop-name)
+                 (translate-property-values val)])
+              properties)
+         (into {}))))
 
 (defn- create-page-uuids
   "Creates maps of unique page names, block contents and property names to their uuids. Used to
@@ -108,7 +113,10 @@
                    (let [property-map {:db/ident k
                                        :logseq.property/type built-in-type}]
                      [property-map v])
-                   (when-let [built-in-type' (get (:build/properties-ref-types new-block) built-in-type)]
+                   (when-let [built-in-type' (get (or (:build/properties-ref-types new-block)
+                                                      ;; Reasonable default for properties like logseq.property/default-value
+                                                      {:entity :number})
+                                                  built-in-type)]
                      (let [property-map {:db/ident k
                                          :logseq.property/type built-in-type'}]
                        [property-map v])))
@@ -122,7 +130,7 @@
                      [property-map v])))))
        (db-property-build/build-property-values-tx-m new-block)))
 
-(defn- extract-content-refs
+(defn- extract-basic-content-refs
   "Extracts basic refs from :block/title like `[[foo]]` or `[[UUID]]`. Can't
   use db-content/get-matched-ids because of named ref support.  Adding more ref
   support would require parsing each block with mldoc and extracting with
@@ -134,7 +142,7 @@
     (map second (re-seq page-ref/page-ref-re s))))
 
 (defn- ->block-tx [{:keys [build/properties] :as m} page-uuids all-idents page-id
-                   {properties-config :properties :keys [build-existing-tx?]}]
+                   {properties-config :properties :keys [build-existing-tx? extract-content-refs?] :as options}]
   (let [build-existing-tx?' (and build-existing-tx? (::existing-block? (meta m)) (not (:build/keep-uuid? m)))
         block (if build-existing-tx?'
                 (select-keys m [:block/uuid])
@@ -143,7 +151,7 @@
                  :block/order (db-order/gen-key nil)
                  :block/parent (or (:block/parent m) {:db/id page-id})})
         pvalue-tx-m (->property-value-tx-m block properties properties-config all-idents)
-        ref-strings (extract-content-refs (:block/title m))]
+        ref-strings (when extract-content-refs? (extract-basic-content-refs (:block/title m)))]
     (cond-> []
       ;; Place property values first since they are referenced by block
       (seq pvalue-tx-m)
@@ -153,7 +161,7 @@
                    (dissoc m :build/properties :build/tags :build/keep-uuid?)
                    (when (seq properties)
                      (->block-properties (merge properties (db-property-build/build-properties-with-ref-values pvalue-tx-m))
-                                         page-uuids all-idents))
+                                         page-uuids all-idents options))
                    (when-let [tags (:build/tags m)]
                      {:block/tags (mapv #(hash-map :db/ident (get-ident all-idents %))
                                         tags)})
@@ -171,7 +179,7 @@
                         :block/refs block-refs})))))))
 
 (defn- build-property-tx
-  [properties page-uuids all-idents property-db-ids
+  [properties page-uuids all-idents property-db-ids options
    [prop-name {:build/keys [property-classes] :as prop-m}]]
   (let [[new-block & additional-tx]
         (if-let [closed-values (seq (map #(merge {:uuid (random-uuid)} %) (:build/closed-values prop-m)))]
@@ -183,14 +191,14 @@
              {:property-attributes
               (merge {:db/id (or (property-db-ids prop-name)
                                  (throw (ex-info "No :db/id for property" {:property prop-name})))}
-                     (select-keys prop-m [:build/properties-ref-types :block/created-at :block/updated-at]))}))
+                     (select-keys prop-m [:build/properties-ref-types :block/created-at :block/updated-at :block/collapsed?]))}))
           [(merge (sqlite-util/build-new-property (get-ident all-idents prop-name)
                                                   (db-property/get-property-schema prop-m)
                                                   {:block-uuid (:block/uuid prop-m)
                                                    :title (:block/title prop-m)})
                   {:db/id (or (property-db-ids prop-name)
                               (throw (ex-info "No :db/id for property" {:property prop-name})))}
-                  (select-keys prop-m [:build/properties-ref-types :block/created-at :block/updated-at]))])
+                  (select-keys prop-m [:build/properties-ref-types :block/created-at :block/updated-at :block/collapsed?]))])
         pvalue-tx-m
         (->property-value-tx-m new-block (:build/properties prop-m) properties all-idents)]
     (cond-> []
@@ -199,9 +207,10 @@
       true
       (conj
        (merge
-        new-block
+        (dissoc new-block :build/properties-ref-types)
         (when-let [props (not-empty (:build/properties prop-m))]
-          (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m)) page-uuids all-idents))
+          (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m))
+                              page-uuids all-idents options))
         (when (seq property-classes)
           {:logseq.property/classes
            (mapv #(hash-map :db/ident (get-ident all-idents %))
@@ -209,7 +218,7 @@
       true
       (into additional-tx))))
 
-(defn- build-properties-tx [properties page-uuids all-idents {:keys [build-existing-tx?]}]
+(defn- build-properties-tx [properties page-uuids all-idents {:keys [build-existing-tx?] :as options}]
   (let [properties' (if build-existing-tx?
                       (->> properties
                            (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/keep-uuid? v)))))
@@ -219,11 +228,11 @@
                              (map #(vector % (new-db-id)))
                              (into {}))
         new-properties-tx (vec
-                           (mapcat (partial build-property-tx properties' page-uuids all-idents property-db-ids)
+                           (mapcat (partial build-property-tx properties' page-uuids all-idents property-db-ids options)
                                    properties'))]
     new-properties-tx))
 
-(defn- build-classes-tx [classes properties-config uuid-maps all-idents {:keys [build-existing-tx?]}]
+(defn- build-classes-tx [classes properties-config uuid-maps all-idents {:keys [build-existing-tx?] :as options}]
   (let [classes' (if build-existing-tx?
                    (->> classes
                         (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/keep-uuid? v)))))
@@ -255,7 +264,8 @@
                              new-block
                              (dissoc class-m :build/properties :build/class-parent :build/class-properties :build/keep-uuid?)
                              (when-let [props (not-empty (:build/properties class-m))]
-                               (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m)) uuid-maps all-idents))
+                               (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m))
+                                                   uuid-maps all-idents options))
                              (when class-parent
                                {:logseq.property/parent
                                 (or (class-db-ids class-parent)
@@ -331,9 +341,11 @@
    [:graph-namespace {:optional true} :keyword]
    [:page-id-fn {:optional true} :any]
    [:auto-create-ontology? {:optional true} :boolean]
-   [:build-existing-tx? {:optional true} :boolean]])
+   [:build-existing-tx? {:optional true} :boolean]
+   [:extract-content-refs? {:optional true} :boolean]
+   [:translate-property-values? {:optional true} :boolean]])
 
-(defn- get-used-properties-from-options
+(defn get-used-properties-from-options
   "Extracts all used properties as a map of properties to their property values. Looks at properties
    from :build/properties and :build/class-properties. Properties from :build/class-properties have
    a ::no-value value"
@@ -366,7 +378,7 @@
 
 ;; TODO: How to detect these idents don't conflict with existing? :db/add?
 (defn- create-all-idents
-  [properties classes graph-namespace]
+  [properties classes {:keys [graph-namespace build-existing-tx?]}]
   (let [property-idents (->> (keys properties)
                              (map #(vector %
                                            (if graph-namespace
@@ -384,11 +396,12 @@
                           (into {}))
         _ (assert (= (count (set (vals class-idents))) (count classes)) "All class db-idents must be unique")
         all-idents (merge property-idents class-idents)]
-    (assert (= (count all-idents) (+ (count property-idents) (count class-idents)))
-            "Class and property db-idents have no overlap")
+    (when-not build-existing-tx?
+      (assert (= (count all-idents) (+ (count property-idents) (count class-idents)))
+              "Class and property db-idents are unique and do not overlap"))
     all-idents))
 
-(defn- build-page-tx [page all-idents page-uuids properties]
+(defn- build-page-tx [page all-idents page-uuids properties options]
   (let [page' (dissoc page :build/tags :build/properties :build/keep-uuid?)
         pvalue-tx-m (->property-value-tx-m page' (:build/properties page) properties all-idents)]
     (cond-> []
@@ -401,8 +414,7 @@
          page'
          (when (seq (:build/properties page))
            (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
-                               page-uuids
-                               all-idents))
+                               page-uuids all-idents options))
          (when-let [tag-idents (->> (:build/tags page) (map #(get-ident all-idents %)) seq)]
            {:block/tags (cond-> (mapv #(hash-map :db/ident %) tag-idents)
                           (empty? (set/intersection (set tag-idents) db-class/page-classes))
@@ -411,7 +423,7 @@
 (defn- build-pages-and-blocks-tx
   [pages-and-blocks all-idents page-uuids {:keys [page-id-fn properties build-existing-tx?]
                                            :or {page-id-fn :db/id}
-                                           :as opts}]
+                                           :as options}]
   (vec
    (mapcat
     (fn [{:keys [page blocks]}]
@@ -432,11 +444,11 @@
          ;; page tx
          (if build-existing-tx?'
            [(select-keys page [:block/uuid :block/created-at :block/updated-at])]
-           (build-page-tx page' all-idents page-uuids properties))
+           (build-page-tx page' all-idents page-uuids properties options))
          ;; blocks tx
          (reduce (fn [acc m]
                    (into acc
-                         (->block-tx m page-uuids all-idents (page-id-fn' page') opts)))
+                         (->block-tx m page-uuids all-idents (page-id-fn' page') options)))
                  []
                  blocks))))
     pages-and-blocks)))
@@ -475,7 +487,7 @@
              (mapcat
               (fn [{:keys [blocks]}]
                 (->> blocks
-                     (mapcat #(extract-content-refs (:block/title %)))
+                     (mapcat #(extract-basic-content-refs (:block/title %)))
                      (remove common-util/uuid-string?)
                      (remove existing-pages))))
              distinct
@@ -524,7 +536,7 @@
 
 (defn- pre-build-pages-and-blocks
   "Pre builds :pages-and-blocks before any indexes like page-uuids are made"
-  [pages-and-blocks properties]
+  [pages-and-blocks properties {:keys [:extract-content-refs?]}]
   (let [ensure-page-uuids (fn [m]
                             (if (get-in m [:page :block/uuid])
                               m
@@ -546,16 +558,20 @@
                                                    (or (:block/uuid page) (common-uuid/gen-uuid :journal-page-uuid date-int))
                                                    :block/tags :logseq.class/Journal})
                                            (with-meta {::new-page? (not (:block/uuid page))})))))
-                           m))]
-    ;; Order matters as some steps depend on previous step having prepared blocks or pages in a certain way
-    (->> pages-and-blocks
-         (add-new-pages-from-properties properties)
-         (map expand-journal)
-         (map expand-block-children)
-         add-new-pages-from-refs
-         ;; This needs to be last to ensure page metadata
-         (map ensure-page-uuids)
-         vec)))
+                           m))
+        ;; Order matters as some steps depend on previous step having prepared blocks or pages in a certain way
+        pages (->> pages-and-blocks
+                   (add-new-pages-from-properties properties)
+                   (map expand-journal)
+                   (map expand-block-children))]
+    (cond->> pages
+      extract-content-refs?
+      add-new-pages-from-refs
+      true
+      ;; This needs to be last to ensure page metadata
+      (map ensure-page-uuids)
+      true
+      vec)))
 
 (defn- infer-property-schema
   "Infers a property schema given a collection of its a property pair values"
@@ -597,13 +613,30 @@
     ;; (when (seq new-classes) (prn :new-classes new-classes))
     {:classes classes' :properties properties'}))
 
+(defn- get-possible-referenced-uuids
+  "Gets all possible ref uuids from either [:block/uuid X] or {:build/journal X}. Uuid scraping
+   is aggressive so some uuids may not be referenced"
+  [input-map]
+  (let [uuids (atom #{})
+        _ (walk/postwalk (fn [f]
+                           ;; This does get a few uuids that aren't :build/keep-uuid? but
+                           ;; that's ok because it consistently gets pvalue uuids
+                           (when (and (vector? f) (= :block/uuid (first f)))
+                             (swap! uuids conj (second f)))
+                           ;; All journals that don't have uuid and could be referenced
+                           (when (and (map? f) (:build/journal f) (not (:block/uuid f)))
+                             (swap! uuids conj (common-uuid/gen-uuid :journal-page-uuid (:build/journal f))))
+                           f)
+                         input-map)]
+    @uuids))
+
 (defn- build-blocks-tx*
-  [{:keys [pages-and-blocks properties graph-namespace auto-create-ontology?]
+  [{:keys [pages-and-blocks properties auto-create-ontology?]
     :as options}]
-  (let [pages-and-blocks' (pre-build-pages-and-blocks pages-and-blocks properties)
+  (let [pages-and-blocks' (pre-build-pages-and-blocks pages-and-blocks properties (dissoc options :pages-and-blocks :properties))
         page-uuids (create-page-uuids pages-and-blocks')
         {:keys [classes properties]} (if auto-create-ontology? (auto-create-ontology options) options)
-        all-idents (create-all-idents properties classes graph-namespace)
+        all-idents (create-all-idents properties classes options)
         properties-tx (build-properties-tx properties page-uuids all-idents options)
         classes-tx (build-classes-tx classes properties page-uuids all-idents options)
         class-ident->id (->> classes-tx (map (juxt :db/ident :db/id)) (into {}))
@@ -628,7 +661,9 @@
       (:build-existing-tx? options)
       (update :init-tx
               (fn [init-tx]
-                (let [indices (mapv #(select-keys % [:block/uuid]) (filter :block/uuid init-tx))]
+                (let [indices
+                      (mapv #(hash-map :block/uuid %)
+                            (get-possible-referenced-uuids {:classes classes :properties properties :pages-and-blocks pages-and-blocks}))]
                   (into indices init-tx)))))))
 
 ;; Public API
@@ -645,6 +680,16 @@
                   (mapcat #(apply-to-block-and-all-children % f) children))))]
     (mapcat #(apply-to-block-and-all-children % f) blocks)))
 
+(defn update-each-block
+  "Calls fn f on each block including all children under :build/children"
+  [blocks f]
+  (mapv (fn [m]
+          (let [updated-m (f m)]
+            (if (:build/children m)
+              (assoc updated-m :build/children (update-each-block (:build/children m) f))
+              updated-m)))
+        blocks))
+
 (defn validate-options
   [{:keys [properties] :as options}]
   (when-let [errors (->> options (m/explain Options) me/humanize)]
@@ -655,9 +700,10 @@
     (let [used-properties (get-used-properties-from-options options)
           undeclared-properties (-> (set (keys used-properties))
                                     (set/difference (set (keys properties)))
-                                    ((fn [x] (remove db-property/logseq-property? x))))]
-      (assert (empty? undeclared-properties)
-              (str "The following properties used in EDN were not declared in :properties: " undeclared-properties)))))
+                                    ((fn [x] (remove db-property/internal-property? x))))]
+      (when (seq undeclared-properties)
+        (throw (ex-info (str "The following properties used in EDN were not declared in :properties: " undeclared-properties)
+                        {:used-properties (select-keys used-properties undeclared-properties)}))))))
 
 (defn ^:large-vars/doc-var build-blocks-tx
   "Given an EDN map for defining pages, blocks and properties, this creates a map
@@ -693,7 +739,8 @@
      * :build/closed-values - Define closed values with a vec of maps. A map contains keys :uuid, :value and :icon.
      * :build/property-classes - Vec of class name keywords. Defines a property's range classes
      * :build/properties-ref-types - Map of internal ref types to public ref types that are valid only for this property.
-       Useful when remapping value ref types e.g. for :logseq.property/default-value
+       Useful when remapping value ref types e.g. for :logseq.property/default-value.
+       Default is `{:entity :number}`
      * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
    * :classes - This is a map to configure classes where the keys are class name keywords
      and the values are maps of datascript attributes e.g. `{:block/title \"Foo\"}`.
@@ -709,6 +756,11 @@
      existing in DB and are skipped for creation. This is useful for building tx on existing DBs e.g. for importing.
      Blocks and pages are updated with any attributes passed to it while all other node types are ignored for update
      unless :build/keep-uuid? is set.
+  * :extract-content-refs? - When set to true, plain text refs e.g. `[[foo]]` are automatically extracted to create pages
+    and to create refs in blocks. This is useful for testing but since it only partially works, not useful for exporting.
+    Default is true
+  * :translate-property-values? - When set to true, property values support special interpretation e.g. `[:build/page ..]`.
+    Default is true
   * :page-id-fn - custom fn that returns ent lookup id for page refs e.g. `[:block/uuid X]`
     Default is :db/id
 
@@ -718,9 +770,10 @@
    supported: :default, :url, :checkbox, :number, :node and :date. :checkbox and
    :number values are written as booleans and integers/floats. :node references
    are written as vectors e.g. `[:build/page {:block/title \"PAGE NAME\"}]`"
-  [options]
-  (validate-options options)
-  (build-blocks-tx* options))
+  [options*]
+  (let [options (merge {:extract-content-refs? true :translate-property-values? true} options*)]
+    (validate-options options)
+    (build-blocks-tx* options)))
 
 (defn create-blocks
   "Builds txs with build-blocks-tx and transacts them. Also provides a shorthand

deps/db/src/logseq/db/sqlite/export.cljs

@@ -2,6 +2,7 @@
   "Builds sqlite.build EDN to represent nodes in a graph-agnostic way.
    Useful for exporting and importing across DB graphs"
   (:require [clojure.set :as set]
+            [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
@@ -10,7 +11,6 @@
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.build :as sqlite-build]))
 
@@ -39,18 +39,18 @@
 
 (defn- buildable-property-value-entity
   "Converts property value to a buildable version"
-  [property-ent pvalue]
-  (cond (ldb/internal-page? pvalue)
+  [property-ent pvalue {:keys [property-value-uuids?]}]
+  (cond (and (not property-value-uuids?) (ldb/internal-page? pvalue))
         ;; Should page properties be pulled here?
         [:build/page (cond-> (shallow-copy-page pvalue)
                        (seq (:block/tags pvalue))
                        (assoc :build/tags (->build-tags (:block/tags pvalue))))]
-        (entity-util/journal? pvalue)
+        (and (not property-value-uuids?) (entity-util/journal? pvalue))
         [:build/page {:build/journal (:block/journal-day pvalue)}]
         :else
-        (if (= :node (:logseq.property/type property-ent))
-          ;; Internal idents take precedence over uuid because they are keep data graph-agnostic
-          (if (some-> pvalue :db/ident db-malli-schema/internal-ident?)
+        (if (contains? #{:node :date} (:logseq.property/type property-ent))
+          ;; Idents take precedence over uuid because they are keep data graph-agnostic
+          (if (:db/ident pvalue)
             (:db/ident pvalue)
             ;; Use metadata distinguish from block references that don't exist like closed values
             ^::existing-property-value? [:block/uuid (:block/uuid pvalue)])
@@ -62,7 +62,7 @@
 (defn- buildable-properties
   "Originally copied from db-test/readable-properties. Modified so that property values are
    valid sqlite.build EDN"
-  [db ent-properties properties-config]
+  [db ent-properties properties-config options]
   (->> ent-properties
        (map (fn [[k v]]
               [k
@@ -74,17 +74,17 @@
                    (throw (ex-info (str "No closed value found for content: " (pr-str (db-property/property-value-content v))) {:properties properties-config})))
                  (cond
                    (de/entity? v)
-                   (buildable-property-value-entity (d/entity db k) v)
+                   (buildable-property-value-entity (d/entity db k) v options)
                    (and (set? v) (every? de/entity? v))
                    (let [property-ent (d/entity db k)]
-                     (set (map (partial buildable-property-value-entity property-ent) v)))
+                     (set (map #(buildable-property-value-entity property-ent % options) v)))
                    :else
                    v))]))
        (into {})))
 
 (defn- build-export-properties
   "The caller of this fn is responsible for building :build/:property-classes unless shallow-copy?"
-  [db user-property-idents {:keys [include-properties? include-uuid? shallow-copy?]}]
+  [db user-property-idents {:keys [include-properties? include-timestamps? include-uuid? shallow-copy?] :as options}]
   (let [properties-config-by-ent
         (->> user-property-idents
              (map (fn [ident]
@@ -93,14 +93,17 @@
                       [property
                        (cond-> (select-keys property
                                             (-> (disj db-property/schema-properties :logseq.property/classes)
-                                                (conj :block/title)))
+                                                (into [:block/title :block/collapsed?])))
                          include-uuid?
-                         (assoc :block/uuid (:block/uuid property))
+                         (assoc :block/uuid (:block/uuid property) :build/keep-uuid? true)
+                         include-timestamps?
+                         (merge (select-keys property [:block/created-at :block/updated-at]))
                          (and (not shallow-copy?) (:logseq.property/classes property))
                          (assoc :build/property-classes (mapv :db/ident (:logseq.property/classes property)))
                          (seq closed-values)
                          (assoc :build/closed-values
-                                (mapv #(cond-> {:value (db-property/property-value-content %) :uuid (random-uuid)}
+                                (mapv #(cond-> {:value (db-property/property-value-content %)
+                                                :uuid (:block/uuid %)}
                                          (:logseq.property/icon %)
                                          (assoc :icon (:logseq.property/icon %)))
                                       closed-values)))])))
@@ -112,25 +115,32 @@
     (if include-properties?
       (->> properties-config-by-ent
            (map (fn [[ent build-property]]
-                  (let [ent-properties (apply dissoc (db-property/properties ent) :block/tags db-property/schema-properties)]
+                  (let [ent-properties (apply dissoc (db-property/properties ent)
+                                              ;; For overlapping class properties, these would be built in :classes
+                                              :logseq.property/parent :logseq.property.class/properties
+                                              (into db-property/schema-properties db-property/public-db-attribute-properties))]
                     [(:db/ident ent)
                      (cond-> build-property
                        (seq ent-properties)
-                       (assoc :build/properties (buildable-properties db ent-properties properties-config)))])))
+                       (assoc :build/properties (buildable-properties db ent-properties properties-config options)))])))
            (into {}))
       properties-config)))
 
 (defn- build-export-class
   "The caller of this fn is responsible for building any classes or properties from this fn
    unless shallow-copy?"
-  [class-ent {:keys [include-parents? include-uuid? shallow-copy?]
+  [class-ent {:keys [include-parents? include-uuid? shallow-copy? include-timestamps?]
               :or {include-parents? true}}]
-  (cond-> (select-keys class-ent [:block/title])
+  (cond-> (select-keys class-ent [:block/title :block/collapsed?])
     include-uuid?
-    (assoc :block/uuid (:block/uuid class-ent))
+    (assoc :block/uuid (:block/uuid class-ent) :build/keep-uuid? true)
+    include-timestamps?
+    (merge (select-keys class-ent [:block/created-at :block/updated-at]))
     (and (:logseq.property.class/properties class-ent) (not shallow-copy?))
     (assoc :build/class-properties
            (mapv :db/ident (:logseq.property.class/properties class-ent)))
+    (and (not shallow-copy?) (:block/alias class-ent))
+    (assoc :block/alias (set (map #(vector :block/uuid (:block/uuid %)) (:block/alias class-ent))))
     ;; It's caller's responsibility to ensure parent is included in final export
     (and include-parents?
          (not shallow-copy?)
@@ -167,7 +177,7 @@
           (into {})))))
 
 (defn- build-node-properties
-  [db entity ent-properties properties]
+  [db entity ent-properties {:keys [properties] :as options}]
   (let [new-user-property-ids (->> (keys ent-properties)
                                    (concat (->> (:block/tags entity)
                                                 (mapcat :logseq.property.class/properties)
@@ -175,28 +185,35 @@
                                    ;; Built-in properties and any possible modifications are not exported
                                    (remove db-property/logseq-property?)
                                    (remove #(get properties %)))]
-    ;; Classes from hare are built in build-node-classes
-    (build-export-properties db new-user-property-ids {})))
+    ;; Classes from here are built in build-node-classes
+    (build-export-properties db new-user-property-ids options)))
 
 (defn- build-node-export
   "Given a block/page entity and optional existing properties, build an export map of its
    tags and properties"
-  [db entity {:keys [properties include-uuid-fn shallow-copy?]
-              :or {include-uuid-fn (constantly false)}}]
-  (let [ent-properties (dissoc (db-property/properties entity) :block/tags)
+  [db entity {:keys [properties include-uuid-fn shallow-copy? include-timestamps? exclude-ontology?]
+              :or {include-uuid-fn (constantly false)}
+              :as options}]
+  (let [ent-properties (apply dissoc (db-property/properties entity) db-property/public-db-attribute-properties)
         build-tags (when (seq (:block/tags entity)) (->build-tags (:block/tags entity)))
-        new-properties (when-not shallow-copy? (build-node-properties db entity ent-properties properties))
+        new-properties (when-not (or shallow-copy? exclude-ontology?)
+                         (build-node-properties db entity ent-properties (dissoc options :shallow-copy? :include-uuid-fn)))
         build-node (cond-> {:block/title (block-title entity)}
+                     (some? (:block/collapsed? entity))
+                     (assoc :block/collapsed? (:block/collapsed? entity))
                      (:block/link entity)
                      (assoc :block/link [:block/uuid (:block/uuid (:block/link entity))])
                      (include-uuid-fn (:block/uuid entity))
                      (assoc :block/uuid (:block/uuid entity) :build/keep-uuid? true)
+                     include-timestamps?
+                     (merge (select-keys entity [:block/created-at :block/updated-at]))
                      (and (not shallow-copy?) (seq build-tags))
                      (assoc :build/tags build-tags)
                      (and (not shallow-copy?) (seq ent-properties))
                      (assoc :build/properties
-                            (buildable-properties db ent-properties (merge properties new-properties))))
-        new-classes (when-not shallow-copy? (build-node-classes db build-node (:block/tags entity) new-properties))]
+                            (buildable-properties db ent-properties (merge properties new-properties) options)))
+        new-classes (when-not (or shallow-copy? exclude-ontology?)
+                      (build-node-classes db build-node (:block/tags entity) new-properties))]
     (cond-> {:node build-node}
       (seq new-classes)
       (assoc :classes new-classes)
@@ -215,7 +232,10 @@
                        (if (set? val-or-vals) val-or-vals [val-or-vals]))))
        set))
 
-(defn- merge-export-maps [& export-maps]
+(defn- merge-export-maps
+  "Merge export maps for partial graph exports. *Do not* use for a full graph
+  export because it makes assumptions about page identity"
+  [& export-maps]
   (let [pages-and-blocks
         (->> (mapcat :pages-and-blocks export-maps)
              ;; TODO: Group by more correct identity for title, same as check-for-existing-entities
@@ -297,15 +317,16 @@
   "Given a vec of block entities, returns the blocks in a sqlite.build EDN format
    and all properties and classes used in these blocks"
   [db blocks {:keys [include-children?] :or {include-children? true} :as opts}]
-  (let [*properties (atom {})
-        *classes (atom {})
+  (let [*properties (atom (or (get-in opts [:graph-ontology :properties]) {}))
+        *classes (atom (or (get-in opts [:graph-ontology :classes]) {}))
         *pvalue-uuids (atom #{})
         id-map (into {} (map (juxt :db/id identity)) blocks)
         children (if include-children? (group-by #(get-in % [:block/parent :db/id]) blocks) {})
         build-block (fn build-block [block*]
                       (let [child-nodes (mapv build-block (get children (:db/id block*) []))
                             {:keys [node properties classes]}
-                            (build-node-export db block* (assoc opts :properties @*properties))
+                            (build-node-export db block* (-> (dissoc opts :graph-ontology)
+                                                             (assoc :properties @*properties)))
                             new-pvalue-uuids (get-pvalue-uuids node)]
                         (when (seq properties) (swap! *properties merge properties))
                         (when (seq classes) (swap! *classes merge classes))
@@ -314,10 +335,12 @@
                           (seq child-nodes) (assoc :build/children child-nodes))))
         roots (remove #(contains? id-map (get-in % [:block/parent :db/id])) blocks)
         exported-blocks (mapv build-block roots)]
-    {:blocks exported-blocks
-     :properties @*properties
-     :classes @*classes
-     :pvalue-uuids @*pvalue-uuids}))
+    (cond-> {:blocks exported-blocks
+             :pvalue-uuids @*pvalue-uuids}
+      (not= @*properties (get-in opts [:graph-ontology :properties]))
+      (assoc :properties @*properties)
+      (not= @*classes (get-in opts [:graph-ontology :classes]))
+      (assoc :classes @*classes))))
 
 (defn- build-uuid-block-export [db pvalue-uuids content-ref-ents {:keys [page-entity]}]
   (let [content-ref-blocks (set (remove entity-util/page? content-ref-ents))
@@ -326,7 +349,7 @@
         uuid-block-pages
         (when (seq uuid-block-ents-to-export)
           (->> uuid-block-ents-to-export
-               (group-by :block/parent)
+               (group-by :block/page)
                ;; Remove page-entity because it's already been built for content-ref-ents
                ;; and it's unlikely and complex to handle for pvalue-uuids
                ((fn [m] (dissoc m page-entity)))
@@ -364,32 +387,56 @@
     (merge {::block (:node node-export)}
            block-export)))
 
-(defn- build-page-blocks-export [db page-entity {:keys [properties classes blocks]}]
-  (let [page-ent-export (build-node-export db page-entity {:properties properties})
-        page (merge (dissoc (:node page-ent-export) :block/title)
-                    (shallow-copy-page page-entity))
+(defn- build-page-blocks-export [db page-entity {:keys [properties classes blocks ontology-page?] :as options}]
+  (let [options' (cond-> (dissoc options :classes :blocks :graph-ontology)
+                   (:exclude-ontology? options)
+                   (assoc :properties (get-in options [:graph-ontology :properties])))
+        page-ent-export (if ontology-page?
+                          ;; Ontology pages are already built in build-graph-ontology-export
+                          {:node (select-keys page-entity [:block/uuid])}
+                          (build-node-export db page-entity options'))
+        page-pvalue-uuids (get-pvalue-uuids (:node page-ent-export))
+        page (if ontology-page?
+               (:node page-ent-export)
+               (merge (dissoc (:node page-ent-export) :block/title)
+                      (shallow-copy-page page-entity)
+                      (when (:block/alias page-entity)
+                        {:block/alias (set (map #(vector :block/uuid (:block/uuid %)) (:block/alias page-entity)))})))
         page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}]
                             :properties properties
                             :classes classes}]
-    (merge-export-maps page-blocks-export page-ent-export)))
+    (assoc (merge-export-maps page-blocks-export page-ent-export)
+           :pvalue-uuids page-pvalue-uuids)))
 
-(defn- build-page-export
-  "Exports page for given page eid"
-  [db eid]
+(defn- get-page-blocks [db eid]
+  (->> (d/datoms db :avet :block/page eid)
+       (map :e)
+       (map #(d/entity db %))))
+
+(defn- build-page-export*
+  [db eid page-blocks* options]
   (let [page-entity (d/entity db eid)
-        datoms (d/datoms db :avet :block/page eid)
-        block-eids (mapv :e datoms)
-        page-blocks* (map #(d/entity db %) block-eids)
-        {:keys [content-ref-uuids content-ref-ents] :as content-ref-export} (build-content-ref-export db page-blocks*)
         page-blocks (->> page-blocks*
                          (sort-by :block/order)
                          ;; Remove property value blocks as they are exported in a block's :build/properties
-                         (remove #(:logseq.property/created-from-property %)))
+                         (remove :logseq.property/created-from-property))
         {:keys [pvalue-uuids] :as blocks-export}
-        (build-blocks-export db page-blocks {:include-uuid-fn content-ref-uuids})
+        (build-blocks-export db page-blocks options)
+        page-blocks-export (build-page-blocks-export db page-entity (merge blocks-export options))
+        page-block-uuids (set/union pvalue-uuids (:pvalue-uuids page-blocks-export))
+        page-export (assoc page-blocks-export :pvalue-uuids page-block-uuids)]
+    page-export))
+
+(defn- build-page-export
+  "Exports page for given page eid"
+  [db eid]
+  (let [page-blocks* (get-page-blocks db eid)
+        {:keys [content-ref-ents] :as content-ref-export} (build-content-ref-export db page-blocks*)
+        {:keys [pvalue-uuids] :as page-export*}
+        (build-page-export* db eid page-blocks* {:include-uuid-fn (:content-ref-uuids content-ref-export)})
+        page-entity (d/entity db eid)
         uuid-block-export (build-uuid-block-export db pvalue-uuids content-ref-ents {:page-entity page-entity})
-        page-blocks-export (build-page-blocks-export db page-entity blocks-export)
-        page-export (finalize-export-maps db page-blocks-export uuid-block-export content-ref-export)]
+        page-export (finalize-export-maps db page-export* uuid-block-export content-ref-export)]
     page-export))
 
 (defn build-view-nodes-export* [db nodes opts]
@@ -403,7 +450,7 @@
         ;; Similar to build-uuid-block-export
         pages-to-blocks
         (->> node-blocks
-             (group-by :block/parent)
+             (group-by :block/page)
              (map (fn [[parent-page-ent blocks]]
                     (merge (build-blocks-export db
                                                 (sort-by :block/order blocks)
@@ -420,7 +467,7 @@
   "Exports given nodes from a view. Nodes are a random mix of blocks and pages"
   [db eids]
   (let [nodes (map #(d/entity db %) eids)
-        property-value-ents (mapcat #(->> (dissoc (db-property/properties %) :block/tags)
+        property-value-ents (mapcat #(->> (apply dissoc (db-property/properties %) db-property/public-db-attribute-properties)
                                           vals
                                           (filter de/entity?))
                                     nodes)
@@ -434,26 +481,36 @@
 
 (defn- build-graph-ontology-export
   "Exports a graph's tags and properties"
-  [db]
-  (let [user-property-idents (d/q '[:find [?db-ident ...]
+  [db {:keys [exclude-namespaces] :as options}]
+  (let [exclude-regex (when (seq exclude-namespaces)
+                        (re-pattern (str "^("
+                                         (string/join "|" (map name exclude-namespaces))
+                                         ")(\\.|$)")))
+        user-property-idents (d/q '[:find [?db-ident ...]
                                     :where [?p :db/ident ?db-ident]
                                     [?p :block/tags :logseq.class/Property]
                                     (not [?p :logseq.property/built-in?])]
                                   db)
-        properties (build-export-properties db user-property-idents {:include-properties? true})
+        user-property-idents' (if (seq exclude-namespaces)
+                                (remove #(re-find exclude-regex (namespace %)) user-property-idents)
+                                user-property-idents)
+        properties (build-export-properties db user-property-idents' (merge options {:include-properties? true}))
         class-ents (->> (d/q '[:find [?class ...]
                                :where [?class :block/tags :logseq.class/Tag]
                                (not [?class :logseq.property/built-in?])]
                              db)
-                        (map #(d/entity db %)))
+                        (map #(d/entity db %))
+                        (remove #(and (seq exclude-namespaces) (re-find exclude-regex (namespace (:db/ident %))))))
         classes
         (->> class-ents
              (map (fn [ent]
-                    (let [ent-properties (dissoc (db-property/properties ent) :block/tags :logseq.property/parent)]
+                    (let [ent-properties (apply dissoc (db-property/properties ent) :logseq.property/parent db-property/public-db-attribute-properties)]
                       (vector (:db/ident ent)
-                              (cond-> (build-export-class ent {})
+                              (cond-> (build-export-class ent options)
                                 (seq ent-properties)
-                                (assoc :build/properties (buildable-properties db ent-properties properties)))))))
+                                (assoc :build/properties
+                                       (-> (buildable-properties db ent-properties properties options)
+                                           (dissoc :logseq.property/classes :logseq.property.class/properties))))))))
              (into {}))]
     (cond-> {}
       (seq properties)
@@ -461,6 +518,156 @@
       (seq classes)
       (assoc :classes classes))))
 
+(defn- get-graph-content-ref-uuids
+  [db {:keys [:exclude-built-in-pages?]}]
+  (let [;; Add support for exclude-built-in-pages? and block-titles as needed
+        block-titles (map :v (d/datoms db :avet :block/title))
+        block-links (if exclude-built-in-pages?
+                      (->> (d/datoms db :avet :block/link)
+                           (keep #(when-not (:logseq.property/built-in? (:block/page (d/entity db (:e %))))
+                                    (:block/uuid (d/entity db (:v %))))))
+                      (->> (d/datoms db :avet :block/link)
+                           (map #(:block/uuid (d/entity db (:v %))))))
+        content-ref-uuids (concat (->> block-titles
+                                       (filter string?)
+                                       (mapcat db-content/get-matched-ids))
+                                  block-links)]
+    (set content-ref-uuids)))
+
+(defn- build-graph-pages-export
+  "Handles pages, journals and their blocks"
+  [db graph-ontology options*]
+  (let [options (merge options*
+                       {:graph-ontology graph-ontology}
+                       ;; dont exclude when ontology is incomplete because :closed values can fail so have to build ontology
+                       (when (empty? (:exclude-namespaces options*))
+                         {:exclude-ontology? true}))
+        page-ids (concat (map :e (d/datoms db :avet :block/tags :logseq.class/Page))
+                         (map :e (d/datoms db :avet :block/tags :logseq.class/Journal)))
+        ontology-ids (set/union (set (map :e (d/datoms db :avet :block/tags :logseq.class/Tag)))
+                                (set (map :e (d/datoms db :avet :block/tags :logseq.class/Property))))
+        page-exports (mapv (fn [eid]
+                             (let [page-blocks (get-page-blocks db eid)]
+                               (build-page-export* db eid page-blocks (merge options {:include-uuid-fn (constantly true)}))))
+                           page-ids)
+        ontology-page-exports
+        (vec
+         (keep (fn [eid]
+                 (when-let [page-blocks (seq (remove :logseq.property/created-from-property (get-page-blocks db eid)))]
+                   (build-page-export* db eid page-blocks (merge options {:include-uuid-fn (constantly true)
+                                                                          :ontology-page? true}))))
+               ontology-ids))
+        page-exports' (remove (fn [page-export]
+                                (and (:exclude-built-in-pages? options)
+                                     (get-in page-export [:pages-and-blocks 0 :page :build/properties :logseq.property/built-in?])))
+                              (concat page-exports ontology-page-exports))
+        alias-uuids  (concat (mapcat (fn [{:keys [pages-and-blocks]}]
+                                       (mapcat #(map second (get-in % [:page :block/alias]))
+                                               pages-and-blocks))
+                                     page-exports')
+                             (mapcat #(map second (:block/alias %))
+                                     (vals (:classes graph-ontology)))
+                             (mapcat #(map second (:block/alias %))
+                                     (vals (:properties graph-ontology))))
+        uuids-to-keep (set/union (set (mapcat :pvalue-uuids page-exports'))
+                                 (set alias-uuids)
+                                 (set (map #(get-in % [:pages-and-blocks 0 :page :block/uuid]) ontology-page-exports)))
+        pages-export {:pages-and-blocks (vec (mapcat :pages-and-blocks page-exports'))
+                      ;; :pvalue-uuids is a misleading name here but using it to keep uuid key consistent across exports
+                      :pvalue-uuids uuids-to-keep}]
+    pages-export))
+
+(defn- build-graph-files
+  [db {:keys [include-timestamps?]}]
+  (->> (d/q '[:find [(pull ?b [:file/path :file/content :file/created-at :file/last-modified-at]) ...]
+              :where [?b :file/path]] db)
+       (mapv #(if include-timestamps?
+                (select-keys % [:file/path :file/content :file/created-at :file/last-modified-at])
+                (select-keys % [:file/path :file/content])))))
+
+(defn- build-kv-values
+  [db]
+  (->> (d/q '[:find [(pull ?b [:db/ident :kv/value]) ...]
+              :where [?b :kv/value]] db)
+       ;; Don't export schema-version as frontend sets this and shouldn't be overridden
+       (remove #(= :logseq.kv/schema-version (:db/ident %)))
+       vec))
+
+(defn remove-uuids-if-not-ref [export-map all-ref-uuids]
+  (let [remove-uuid-if-not-ref (fn [m] (if (contains? all-ref-uuids (:block/uuid m))
+                                         m
+                                         (dissoc m :block/uuid :build/keep-uuid?)))]
+    (-> export-map
+        (update :classes update-vals remove-uuid-if-not-ref)
+        (update :properties update-vals remove-uuid-if-not-ref)
+        (update :pages-and-blocks
+                (fn [pages-and-blocks]
+                  (mapv (fn [{:keys [page blocks]}]
+                          {:page (remove-uuid-if-not-ref page)
+                           :blocks (sqlite-build/update-each-block blocks remove-uuid-if-not-ref)})
+                        pages-and-blocks))))))
+
+(defn sort-pages-and-blocks
+  "Provide a reliable sort order since this tends to be large. Helps with diffing
+   and readability"
+  [pages-and-blocks]
+  (vec
+   (sort-by #(or (get-in % [:page :block/title])
+                 (some-> (get-in % [:page :build/journal]) str)
+                 (str (get-in % [:page :block/uuid])))
+            pages-and-blocks)))
+
+(defn- add-ontology-for-include-namespaces
+  "Adds :properties to export for given namespace parents. Current use case is for :exclude-namespaces
+   so no need to add :classes yet"
+  [db {::keys [auto-include-namespaces] :as graph-export}]
+  (let [include-regex (re-pattern (str "^("
+                                       (string/join "|" (map name auto-include-namespaces))
+                                       ")(\\.|$)"))
+        used-properties
+        (->> (sqlite-build/get-used-properties-from-options graph-export)
+             keys
+             (remove db-property/internal-property?)
+             (filter #(re-find include-regex (namespace %)))
+             (map #(vector % (select-keys (d/entity db %) [:logseq.property/type :db/cardinality])))
+             (into {}))]
+    (-> (merge-export-maps (select-keys graph-export [:properties])
+                           {:properties used-properties})
+        (select-keys [:properties]))))
+
+(defn- build-graph-export
+  "Exports whole graph. Has the following options:
+   * :include-timestamps? - When set, timestamps are included on all blocks
+   * :exclude-namespaces - A set of parent namespaces to exclude from properties and classes.
+     This is useful for graphs seeded with an ontology e.g. schema.org as it eliminates noisy and needless
+     export+import
+   * :exclude-built-in-pages? - When set, built-in pages are excluded from export
+   * :exclude-files? - When set, files are excluded from export"
+  [db {:keys [exclude-files?] :as options*}]
+  (let [options (merge options* {:property-value-uuids? true})
+        content-ref-uuids (get-graph-content-ref-uuids db options)
+        ontology-options (merge options {:include-uuid? true})
+        ontology-export (build-graph-ontology-export db ontology-options)
+        ontology-pvalue-uuids (set (concat (mapcat get-pvalue-uuids (vals (:properties ontology-export)))
+                                           (mapcat get-pvalue-uuids (vals (:classes ontology-export)))))
+        pages-export (build-graph-pages-export db ontology-export options)
+        graph-export* (-> (merge ontology-export pages-export) (dissoc :pvalue-uuids))
+        graph-export (if (seq (:exclude-namespaces options))
+                       (assoc graph-export* ::auto-include-namespaces (:exclude-namespaces options))
+                       graph-export*)
+        all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export))
+        files (when-not exclude-files? (build-graph-files db options))
+        kv-values (build-kv-values db)
+        ;; Remove all non-ref uuids after all nodes are built.
+        ;; Only way to ensure all pvalue uuids present across block types
+        graph-export' (-> (remove-uuids-if-not-ref graph-export all-ref-uuids)
+                          (update :pages-and-blocks sort-pages-and-blocks))]
+    (cond-> graph-export'
+      (not exclude-files?)
+      (assoc ::graph-files files)
+      true
+      (assoc ::kv-values kv-values))))
+
 (defn- find-undefined-classes-and-properties [{:keys [classes properties pages-and-blocks]}]
   (let [referenced-classes
         (->> (concat (mapcat :build/property-classes (vals properties))
@@ -474,7 +681,7 @@
         (->> (concat (mapcat :build/class-properties (vals classes))
                      (mapcat (comp keys :build/properties :page) pages-and-blocks)
                      (mapcat #(sqlite-build/extract-from-blocks (:blocks %) (comp keys :build/properties)) pages-and-blocks))
-             (remove db-property/logseq-property?)
+             (remove db-property/internal-property?)
              set)
         undefined-properties (set/difference referenced-properties (set (keys properties)))
         undefined (cond-> {}
@@ -492,7 +699,8 @@
                      (mapcat #(sqlite-build/extract-from-blocks (:blocks %) (fn [m] (some-> m :block/uuid vector)))
                              pages-and-blocks))
              set)
-        ;; only looks one-level deep in properties e.g. not inside :build/page
+        ;; Only looks one-level deep in properties e.g. not inside :build/page
+        ;; Doesn't find :block/link refs
         ref-uuids
         (->> (concat (mapcat get-pvalue-uuids (vals classes))
                      (mapcat get-pvalue-uuids (vals properties))
@@ -503,11 +711,14 @@
 
 (defn- ensure-export-is-valid
   "Checks that export map is usable by sqlite.build including checking that
-   all referenced properties and classes are defined"
-  [export-map]
-  (sqlite-build/validate-options export-map)
+   all referenced properties and classes are defined. Checks related to properties and
+   classes are disabled when :exclude-namespaces is set because those checks can't be done"
+  [export-map {:keys [graph-options]}]
+  (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map))
   (let [undefined-uuids (find-undefined-uuids export-map)
-        undefined (cond-> (find-undefined-classes-and-properties export-map)
+        undefined (cond-> {}
+                    (empty? (:exclude-namespaces graph-options))
+                    (merge (find-undefined-classes-and-properties export-map))
                     (seq undefined-uuids)
                     (assoc :uuids undefined-uuids))]
     (when (seq undefined)
@@ -526,9 +737,16 @@
           :view-nodes
           (build-view-nodes-export db (:node-ids options))
           :graph-ontology
-          (build-graph-ontology-export db))]
-    (ensure-export-is-valid (dissoc export-map ::block))
-    export-map))
+          (build-graph-ontology-export db {})
+          :graph
+          (build-graph-export db (:graph-options options)))]
+    (if (get-in options [:graph-options :catch-validation-errors?])
+      (try
+        (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values) options)
+        (catch ExceptionInfo e
+          (println "Caught error:" e)))
+      (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values) options))
+    (assoc export-map ::export-type export-type)))
 
 ;; Import fns
 ;; ==========
@@ -548,9 +766,10 @@
 (defn- check-for-existing-entities
   "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import.
    Also checks for property conflicts between existing properties and properties to be imported"
-  [db {:keys [pages-and-blocks classes properties]} property-conflicts]
+  [db {:keys [pages-and-blocks classes properties] ::keys [export-type] :as export-map} property-conflicts]
   (let [export-map
-        (cond-> {:build-existing-tx? true}
+        (cond-> {:build-existing-tx? true
+                 :extract-content-refs? false}
           (seq pages-and-blocks)
           (assoc :pages-and-blocks
                  (mapv (fn [m]
@@ -578,13 +797,20 @@
                                            :expected (select-keys ent [:logseq.property/type :db/cardinality])}))
                                  [k (assoc v :block/uuid (:block/uuid ent))])
                                [k v])))
-                      (into {}))))
-        export-map'
-        (walk/postwalk (fn [f]
-                         (if (and (vector? f) (= :build/page (first f)))
-                           [:build/page (add-uuid-to-page-if-exists db (second f))]
-                           f))
-                       export-map)]
+                      (into {})))
+          ;; Graph export doesn't use :build/page so this speeds up build
+          (= :graph export-type)
+          (assoc :translate-property-values? false)
+          (= :graph export-type)
+          ;; Currently all graph-files are created by app so no need to distinguish between user and built-in ones yet
+          (merge (dissoc export-map :pages-and-blocks :classes :properties)))
+        export-map' (if (= :graph export-type)
+                      export-map
+                      (walk/postwalk (fn [f]
+                                       (if (and (vector? f) (= :build/page (first f)))
+                                         [:build/page (add-uuid-to-page-if-exists db (second f))]
+                                         f))
+                                     export-map))]
     export-map'))
 
 (defn- build-block-import-options
@@ -599,16 +825,36 @@
     (merge-export-maps export-map {:pages-and-blocks pages-and-blocks})))
 
 (defn build-import
-  "Given an entity's export map, build the import tx to create it"
+  "Given an entity's export map, build the import tx to create it. In addition to standard sqlite.build keys,
+   an export map can have the following namespaced keys:
+   * ::export-type - Keyword indicating export type
+   * ::block - Block map for a :block export
+   * ::graph-files - Vec of files for a :graph export
+   * ::kv-values - Vec of :kv/value maps for a :graph export
+   * ::auto-include-namespaces - A set of parent namespaces to include from properties and classes
+     for a :graph export. See :exclude-namespaces in build-graph-export for a similar option
+
+   This fn then returns a map of txs to transact with the following keys:
+   * :init-tx - Txs that must be transacted first, usually because they define new properties
+   * :block-props-tx - Txs to transact after :init-tx, usually because they use newly defined properties
+   * :misc-tx - Txs to transact unrelated to other txs"
   [export-map* db {:keys [current-block]}]
   (let [export-map (if (and (::block export-map*) current-block)
                      (build-block-import-options current-block export-map*)
                      export-map*)
+        export-map' (if (and (= :graph (::export-type export-map*)) (seq (::auto-include-namespaces export-map*)))
+                      (merge (dissoc export-map :properties ::auto-include-namespaces)
+                             (add-ontology-for-include-namespaces db export-map))
+                      export-map)
         property-conflicts (atom [])
-        export-map' (check-for-existing-entities db export-map property-conflicts)]
+        export-map'' (check-for-existing-entities db export-map' property-conflicts)]
     (if (seq @property-conflicts)
       (do
         (js/console.error :property-conflicts @property-conflicts)
         {:error (str "The following imported properties conflict with the current graph: "
                      (pr-str (mapv :property-id @property-conflicts)))})
-      (sqlite-build/build-blocks-tx export-map'))))
+      (if (= :graph (::export-type export-map''))
+        (-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files ::kv-values ::export-type))
+            (assoc :misc-tx (vec (concat (::graph-files export-map'')
+                                         (::kv-values export-map'')))))
+        (sqlite-build/build-blocks-tx export-map'')))))

deps/db/src/logseq/db/sqlite/util.cljs

@@ -43,6 +43,7 @@
       (try (transit/write writer o)
            (catch :default e
              (prn ::write-transit-str o)
+             (js/console.trace)
              (throw e))))))
 
 (def read-transit-str
@@ -50,7 +51,13 @@
                                    "datascript/Entity" identity)
                             (merge read-handlers))
         reader (transit/reader :json {:handlers read-handlers*})]
-    (fn read-transit-str* [s] (transit/read reader s))))
+    (fn read-transit-str* [s]
+      ;; TODO: delete the following pred later
+      ;; https://github.com/logseq/logseq/pull/11790#discussion_r2014120469
+      (if (and (string? s) (identical? "[" (first s)))
+        (transit/read reader s)
+        (do (prn :invalid-transit-string s)
+            s)))))
 
 (defn db-based-graph?
   [graph-name]

deps/db/test/logseq/db/sqlite/build_test.cljs

@@ -179,4 +179,30 @@
     (is (contains? (:block/refs block-with-property-ref) (d/entity @conn :user.property/p1)))
 
     (is (= block-uuid (:block/uuid (db-test/find-block-by-content @conn "hi"))))
-    (is (contains? (:block/refs block-with-block-ref) (db-test/find-block-by-content @conn "hi")))))
+    (is (contains? (:block/refs block-with-block-ref) (db-test/find-block-by-content @conn "hi")))))
+
+(deftest build-class-and-property-pages
+  (let [class-uuid (random-uuid)
+        property-uuid (random-uuid)
+        conn (db-test/create-conn-with-blocks
+              {:classes {:C1 {:block/uuid class-uuid :build/keep-uuid? true}}
+               :properties {:p1 {:block/uuid property-uuid :build/keep-uuid? true}}
+               :pages-and-blocks
+               [{:page {:block/uuid class-uuid}
+                 :blocks [{:block/title "b1"
+                           :build/children [{:block/title "b2"}]}]}
+                {:page {:block/uuid property-uuid}
+                 :blocks [{:block/title "b3"
+                           :build/children [{:block/title "b4"}]}]}]
+               :build-existing-tx? true})]
+    (is (= ["b1" "b2"]
+           (->> (d/q '[:find [?b ...] :in $ ?page-id :where [?b :block/page ?page-id]]
+                     @conn [:block/uuid class-uuid])
+                (map #(:block/title (d/entity @conn %)))))
+        "Class page has correct blocks")
+
+    (is (= ["b3" "b4"]
+           (->> (d/q '[:find [?b ...] :in $ ?page-id :where [?b :block/page ?page-id]]
+                     @conn [:block/uuid property-uuid])
+                (map #(:block/title (d/entity @conn %)))))
+        "Property page has correct blocks")))

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -1,9 +1,14 @@
 (ns logseq.db.sqlite.export-test
   (:require [cljs.pprint]
             [cljs.test :refer [deftest is testing]]
+            [clojure.walk :as walk]
             [datascript.core :as d]
+            [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
+            [logseq.common.uuid :as common-uuid]
+            [logseq.db :as ldb]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.test.helper :as db-test]
@@ -35,6 +40,54 @@
     (sqlite-export/build-export @import-conn {:export-type :block
                                               :block-id (:db/id import-block)})))
 
+(defn- export-page-and-import-to-another-graph
+  "Exports given page from one graph/conn, imports it to a 2nd graph, validates
+  it and then exports the page from the 2nd graph"
+  [export-conn import-conn page-title]
+  (let [page (db-test/find-page-by-title @export-conn page-title)
+        {:keys [init-tx block-props-tx] :as _txs}
+        (-> (sqlite-export/build-export @export-conn {:export-type :page :page-id (:db/id page)})
+            ;; ((fn [x] (cljs.pprint/pprint {:export x}) x))
+            (sqlite-export/build-import @import-conn {}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)
+        _ (validate-db @import-conn)
+        page2 (db-test/find-page-by-title @import-conn page-title)]
+    (sqlite-export/build-export @import-conn {:export-type :page :page-id (:db/id page2)})))
+
+(defn- import-second-time-assertions [conn conn2 page-title original-data
+                                      & {:keys [transform-expected-blocks]
+                                         :or {transform-expected-blocks (fn [bs] (into bs bs))}}]
+  (let [page (db-test/find-page-by-title @conn2 page-title)
+        imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
+        updated-page (db-test/find-page-by-title @conn2 page-title)
+        expected-page-and-blocks
+        (update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks)]
+
+    ;; Assume first page is one being imported for now
+    (is (= (first expected-page-and-blocks)
+           (first (:pages-and-blocks imported-page)))
+        "Blocks are appended to existing page")
+    (is (= (:block/created-at page) (:block/created-at updated-page))
+        "Existing page didn't get re-created")
+    (is (= (:block/updated-at page) (:block/updated-at updated-page))
+        "Existing page didn't get updated")))
+
+(defn- export-graph-and-import-to-another-graph
+  "Exports graph and imports it to a 2nd graph, validates it and then exports the 2nd graph"
+  [export-conn import-conn export-options]
+  (let [{:keys [init-tx block-props-tx misc-tx] :as _txs}
+        (-> (sqlite-export/build-export @export-conn {:export-type :graph :graph-options export-options})
+            (sqlite-export/build-import @import-conn {}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)
+        _ (d/transact! import-conn misc-tx)
+        _ (validate-db @import-conn)
+        imported-graph (sqlite-export/build-export @import-conn {:export-type :graph :graph-options export-options})]
+    imported-graph))
+
 (defn- expand-properties
   "Add default values to properties of an input export map to test against a
   db-based export map"
@@ -61,6 +114,8 @@
                  (assoc :block/title (name k)))]))
        (into {})))
 
+(def sort-pages-and-blocks sqlite-export/sort-pages-and-blocks)
+
 ;; Tests
 ;; =====
 
@@ -173,40 +228,6 @@
            (first (:pages-and-blocks imported-block)))
         "Imported page equals exported page of page ref")))
 
-(defn- export-page-and-import-to-another-graph
-  "Exports given page from one graph/conn, imports it to a 2nd graph, validates
-  it and then exports the page from the 2nd graph"
-  [export-conn import-conn page-title]
-  (let [page (db-test/find-page-by-title @export-conn page-title)
-        {:keys [init-tx block-props-tx] :as _txs}
-        (-> (sqlite-export/build-export @export-conn {:export-type :page :page-id (:db/id page)})
-            ;; ((fn [x] (cljs.pprint/pprint {:export x}) x))
-            (sqlite-export/build-import @import-conn {}))
-        ;; _ (cljs.pprint/pprint _txs)
-        _ (d/transact! import-conn init-tx)
-        _ (d/transact! import-conn block-props-tx)
-        _ (validate-db @import-conn)
-        page2 (db-test/find-page-by-title @import-conn page-title)]
-    (sqlite-export/build-export @import-conn {:export-type :page :page-id (:db/id page2)})))
-
-(defn- import-second-time-assertions [conn conn2 page-title original-data
-                                      & {:keys [transform-expected-blocks]
-                                         :or {transform-expected-blocks (fn [bs] (into bs bs))}}]
-  (let [page (db-test/find-page-by-title @conn2 page-title)
-        imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
-        updated-page (db-test/find-page-by-title @conn2 page-title)
-        expected-page-and-blocks
-        (update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks)]
-
-    ;; Assume first page is one being imported for now
-    (is (= (first expected-page-and-blocks)
-           (first (:pages-and-blocks imported-page)))
-        "Blocks are appended to existing page")
-    (is (= (:block/created-at page) (:block/created-at updated-page))
-        "Existing page didn't get re-created")
-    (is (= (:block/updated-at page) (:block/updated-at updated-page))
-        "Existing page didn't get updated")))
-
 ;; Tests a variety of blocks including block children with new properties, blocks with users classes
 ;; and blocks with built-in properties and classes
 (deftest import-page-with-different-blocks
@@ -262,9 +283,11 @@
                        :build/property-classes [:user.class/NodeClass]}
                       :user.property/p2
                       {:logseq.property/type :default}}
+         :extract-content-refs? false
          :pages-and-blocks
          [{:page {:block/title "page1"}
            :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}
+                    {:block/title (str "not a page ref `" (page-ref/->page-ref "foo") "`")}
                     {:block/title (str "block ref to " (page-ref/->page-ref block-uuid))}
                     {:block/title "ref in properties"
                      :build/properties {:user.property/p2 (str "pvalue ref to " (page-ref/->page-ref pvalue-page-uuid))}}
@@ -506,4 +529,240 @@
 
     (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-nodes)))
     (is (= (expand-properties (:properties original-data)) (:properties imported-nodes)))
-    (is (= (expand-classes (:classes original-data)) (:classes imported-nodes)))))
+    (is (= (expand-classes (:classes original-data)) (:classes imported-nodes)))))
+
+(defn- build-original-graph-data
+  [& {:keys [exclude-namespaces?]}]
+  (let [internal-block-uuid (random-uuid)
+        favorited-uuid (random-uuid)
+        block-pvalue-uuid (random-uuid)
+        property-pvalue-uuid (random-uuid)
+        page-pvalue-uuid (random-uuid)
+        page-object-uuid (random-uuid)
+        page-alias-uuid (random-uuid)
+        closed-value-uuid (random-uuid)
+        property-uuid (random-uuid)
+        class-uuid (random-uuid)
+        class-alias-uuid (random-uuid)
+        class2-uuid (random-uuid)
+        journal-uuid (common-uuid/gen-uuid :journal-page-uuid 19650201)
+        original-data
+        {:properties
+         {:user.property/num {:logseq.property/type :number
+                              :block/uuid property-uuid
+                              :build/keep-uuid? true
+                              :block/collapsed? true
+                              :build/properties (if exclude-namespaces?
+                                                  {}
+                                                  {:user.property/node #{[:block/uuid property-pvalue-uuid]}
+                                                   :logseq.property/default-value 42})}
+          :user.property/default-closed
+          {:logseq.property/type :default
+           :build/closed-values [{:value "joy" :uuid closed-value-uuid}
+                                 {:value "sad" :uuid (random-uuid)}]}
+          :user.property/checkbox {:logseq.property/type :checkbox}
+          :user.property/date {:logseq.property/type :date}
+          :user.property/url {:logseq.property/type :url
+                              :build/properties {:logseq.property/description "desc for url"}}
+          :user.property/node {:logseq.property/type :node
+                               :db/cardinality :db.cardinality/many
+                               :build/property-classes [:user.class/MyClass]}}
+         :classes
+         {:user.class/MyClass (cond-> {:build/properties {:user.property/url "https://example.com/MyClass"}
+                                       :block/uuid class-uuid
+                                       :build/keep-uuid? true}
+                                (not exclude-namespaces?)
+                                (assoc :block/alias #{[:block/uuid class-alias-uuid]}))
+          :user.class/MyClassAlias {:block/uuid class-alias-uuid
+                                    :build/keep-uuid? true}
+          :user.class/MyClass2 {:build/class-parent :user.class/MyClass
+                                :block/collapsed? true
+                                :block/uuid class2-uuid
+                                :build/keep-uuid? true
+                                :build/properties {:logseq.property/description "tests child class"}}}
+         :pages-and-blocks
+         [{:page {:block/title "page1"
+                  :block/uuid favorited-uuid :build/keep-uuid? true
+                  :build/properties {:user.property/checkbox false
+                                     :user.property/node #{[:block/uuid page-pvalue-uuid]}}}
+           :blocks [{:block/title "b1"
+                     :build/properties {:user.property/num 1
+                                        :user.property/default-closed [:block/uuid closed-value-uuid]
+                                        :user.property/date [:block/uuid journal-uuid]}}
+                    {:block/title "b2" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}}
+                    {:block/title "b3" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}}]}
+          {:page {:block/title "page object"
+                  :block/uuid page-object-uuid
+                  :build/keep-uuid? true}
+           :blocks []}
+          {:page {:block/title "page2" :build/tags [:user.class/MyClass2]}
+           :blocks [{:block/title "hola" :block/uuid internal-block-uuid :build/keep-uuid? true}
+                    {:block/title "myclass object 1"
+                     :build/tags [:user.class/MyClass]
+                     :block/uuid block-pvalue-uuid
+                     :build/keep-uuid? true}
+                    (cond-> {:block/title "myclass object 2"
+                             :build/tags [:user.class/MyClass]}
+                      (not exclude-namespaces?)
+                      (merge {:block/uuid property-pvalue-uuid
+                              :build/keep-uuid? true}))
+                    {:block/title "myclass object 3"
+                     :build/tags [:user.class/MyClass]
+                     :block/uuid page-pvalue-uuid
+                     :build/keep-uuid? true}
+                    {:block/title "ref blocks"
+                     :block/collapsed? true
+                     :build/children
+                     [{:block/title (str "internal block ref to " (page-ref/->page-ref internal-block-uuid))}
+                      {:block/title "node block"
+                       :build/properties {:user.property/node #{[:block/uuid block-pvalue-uuid]}}}
+                      {:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}
+                      {:block/title (str "class ref to " (page-ref/->page-ref class-uuid))}]}]}
+          {:page {:block/title "Alias for 2/28" :block/uuid page-alias-uuid :build/keep-uuid? true}
+           :blocks []}
+          {:page {:build/journal 20250228
+                  :block/alias #{[:block/uuid page-alias-uuid]}
+                  :build/properties {:user.property/num 1}}
+           :blocks [{:block/title "journal block"}]}
+          {:page {:build/journal 19650201
+                  :block/uuid journal-uuid
+                  :build/keep-uuid? true}
+           :blocks []}
+          {:page {:block/uuid class-uuid}
+           :blocks [{:block/title "class block1"
+                     :build/children [{:block/title "class block2"}]}]}
+          {:page {:block/uuid class2-uuid}
+           :blocks [{:block/title "class2 block1"}]}
+          {:page {:block/uuid property-uuid}
+           :blocks [{:block/title "property block1"}]}
+          ;; built-in pages
+          {:page {:block/title "Contents" :build/properties {:logseq.property/built-in? true}}
+           :blocks [{:block/title "right sidebar"}]}
+          {:page {:block/title common-config/favorites-page-name
+                  :build/properties {:logseq.property/built-in? true, :logseq.property/hide? true}}
+           :blocks [(ldb/build-favorite-tx favorited-uuid)]}
+          {:page {:block/title common-config/views-page-name
+                  :build/properties {:logseq.property/built-in? true, :logseq.property/hide? true}}
+           :blocks [{:block/title "All"
+                     :build/properties {:logseq.property/view-for :logseq.class/Task
+                                        :logseq.property.view/feature-type :class-objects}}
+                    {:block/title "All"
+                     :build/properties {:logseq.property/view-for :user.class/MyClass
+                                        :logseq.property.view/feature-type :class-objects}}
+                    {:block/title "Linked references",
+                     :build/properties
+                     {:logseq.property.view/type :logseq.property.view/type.list,
+                      :logseq.property.view/feature-type :linked-references,
+                      :logseq.property/view-for [:block/uuid journal-uuid]}}]}]
+         ::sqlite-export/graph-files
+         [{:file/path "logseq/config.edn"
+           :file/content "{:foo :bar}"}
+          {:file/path "logseq/custom.css"
+           :file/content ".foo {background-color: blue}"}
+          {:file/path "logseq/custom.js"
+           :file/content "// comment"}]
+         :build-existing-tx? true}]
+    original-data))
+
+(deftest import-graph
+  (let [original-data (build-original-graph-data)
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        ;; set to an unobtainable version to test this ident
+        _ (d/transact! conn [{:db/ident :logseq.kv/schema-version :kv/value {:major 1 :minor 0}}])
+        original-kv-values (remove #(= :logseq.kv/schema-version (:db/ident %))
+                                   (d/q '[:find [(pull ?b [:db/ident :kv/value]) ...] :where [?b :kv/value]] @conn))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn)
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {})]
+
+    ;; (cljs.pprint/pprint (set (:pages-and-blocks original-data)))
+    ;; (cljs.pprint/pprint (set (:pages-and-blocks imported-graph)))
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (sort-pages-and-blocks (:pages-and-blocks original-data))
+    ;;                                                 (:pages-and-blocks imported-graph))))
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= 1 (count (d/datoms @conn2 :avet :block/title "page object")))
+        "No duplicate pages for pvalue uuids used more than once")
+    (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
+    (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")
+    (is (= original-kv-values (::sqlite-export/kv-values imported-graph))
+        "All :kv/value entities are imported except for ignored ones")
+    (is (not= (:kv/value (d/entity @conn :logseq.kv/schema-version))
+              (:kv/value (d/entity @conn2 :logseq.kv/schema-version)))
+        "Ignored :kv/value is not updated")))
+
+(deftest import-graph-with-timestamps
+  (let [original-data* (build-original-graph-data)
+        original-data (-> original-data*
+                          (update :pages-and-blocks
+                                  (fn [pages-and-blocks]
+                                    (walk/postwalk (fn [e]
+                                                     (if (and (map? e) (or (:block/title e) (:build/journal e)))
+                                                       (common-util/block-with-timestamps e)
+                                                       e))
+                                                   pages-and-blocks)))
+                          (update :classes update-vals common-util/block-with-timestamps)
+                          (update :properties update-vals common-util/block-with-timestamps)
+                          (update ::sqlite-export/graph-files
+                                  (fn [files]
+                                    (mapv #(let [now (js/Date.)]
+                                             (merge % {:file/created-at now :file/last-modified-at now}))
+                                          files))))
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn)
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {:include-timestamps? true})]
+
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (sort-pages-and-blocks (:pages-and-blocks original-data))
+    ;;                                                 (:pages-and-blocks imported-graph))))
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
+    (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")))
+
+(deftest import-graph-with-exclude-namespaces
+  (let [original-data (build-original-graph-data {:exclude-namespaces? true})
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn-with-blocks
+               {:properties (update-vals (:properties original-data) #(dissoc % :build/properties))
+                :classes (update-vals (:classes original-data) #(dissoc % :build/properties))})
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {:exclude-namespaces #{:user}})]
+
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (sort-pages-and-blocks (:pages-and-blocks original-data))
+    ;;                                                 (:pages-and-blocks imported-graph))))
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")))
+
+(deftest import-graph-with-overlapping-ontology-properties
+  (let [overlapping-uuid (random-uuid)
+        original-data
+        {:properties {:user.property/p1
+                      {:block/uuid overlapping-uuid
+                       :build/keep-uuid? true
+                       :logseq.property/type :node
+                       :build/property-classes [:user.property/p1]}
+                      :user.property/p2 {:logseq.property/type :default}}
+         :classes {:user.class/C1 {}
+                   :user.property/p1
+                   {:build/keep-uuid? true
+                    :block/uuid overlapping-uuid
+                    :build/class-parent :user.class/C1
+                    :build/class-properties [:user.property/p2]}}
+         :pages-and-blocks
+         [{:page {:block/uuid overlapping-uuid}
+           :blocks [{:block/title "b1"
+                     :build/children [{:block/title "b2"}]}]}]
+         :build-existing-tx? true}
+        conn (db-test/create-conn-with-blocks original-data)
+        conn2 (db-test/create-conn)
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {:exclude-built-in-pages? true})]
+
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
+    (is (= (expand-classes (:classes original-data))
+           (-> (:classes imported-graph)
+               (medley/dissoc-in [:user.property/p1 :build/properties]))))))

deps/graph-parser/script/db_import.cljs

@@ -125,6 +125,17 @@
       (p/let [_ (gp-exporter/export-doc-files conn files' <read-file doc-options)]
         {:import-state (:import-state doc-options)}))))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (def spec
   "Options spec"
   {:help {:alias :h
@@ -160,10 +171,7 @@
             (println (str "Usage: $0 FILE-GRAPH DB-GRAPH [OPTIONS]\nOptions:\n"
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
-        [dir db-name] (if (string/includes? db-graph-dir "/")
-                        (let [graph-dir' (resolve-path db-graph-dir)]
-                          ((juxt node-path/dirname node-path/basename) graph-dir'))
-                        [(node-path/join (os/homedir) "logseq" "graphs") db-graph-dir])
+        [dir db-name] (get-dir-and-db-name db-graph-dir)
         file-graph' (resolve-path file-graph)
         conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath)})
         directory? (.isDirectory (fs/statSync file-graph'))

deps/outliner/script/transact.cljs

@@ -10,15 +10,24 @@
             [logseq.outliner.db-pipeline :as db-pipeline]
             [nbb.core :as nbb]))
 
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
 (defn -main [args]
   (when (< (count args) 3)
     (println "Usage: $0 GRAPH-DIR QUERY TRANSACT-FN")
     (js/process.exit 1))
   (let [[graph-dir query* transact-fn*] args
         dry-run? (contains? (set args) "-n")
-        [dir db-name] (if (string/includes? graph-dir "/")
-                        ((juxt node-path/dirname node-path/basename) graph-dir)
-                        [(node-path/join (os/homedir) "logseq" "graphs") graph-dir])
+        [dir db-name] (get-dir-and-db-name graph-dir)
         conn (sqlite-cli/open-db! dir db-name)
         ;; find blocks to update
         query (into (edn/read-string query*) [:in '$ '%]) ;; assumes no :in are in queries

deps/shui/src/logseq/shui/dialog/core.cljs

@@ -1,11 +1,11 @@
 (ns logseq.shui.dialog.core
-  (:require [rum.core :as rum]
-            [daiquiri.interpreter :refer [interpret]]
-            [medley.core :as medley]
-            [logseq.shui.util :as util]
+  (:require [daiquiri.interpreter :refer [interpret]]
             [logseq.shui.base.core :as base]
             [logseq.shui.form.core :as form]
-            [promesa.core :as p]))
+            [logseq.shui.util :as util]
+            [medley.core :as medley]
+            [promesa.core :as p]
+            [rum.core :as rum]))
 
 ;; provider
 (def dialog (util/lsui-wrap "Dialog"))
@@ -127,7 +127,7 @@
                 auto-width? close-btn? root-props content-props]} config
         props (dissoc config
                       :id :title :description :content :footer :auto-width? :close-btn?
-                      :align :on-open-change :open? :root-props :content-props)
+                      :close :align :on-open-change :open? :root-props :content-props)
         props (assoc-in props [:overlay-props :data-align] (name (or align :center)))]
 
     (rum/use-effect!

e2e-tests/logseq-api.spec.ts

@@ -215,7 +215,7 @@ test('(DB graph): block related apis',
     prop1 = await callAPI('get_property', 'map1')
     const b1p = await callAPI('get_block_property', b1.uuid, 'map1')
 
-    expect(prop1.schema.type).toBe('map')
+    expect(prop1.type).toBe('map')
     expect(b1p).toEqual({a: 1})
 
     // await page.pause()

libs/src/LSPlugin.caller.ts

@@ -50,6 +50,7 @@ class LSPluginCaller extends EventEmitter {
     }
   }
 
+  // run in host
   async connectToChild() {
     if (this._connected) return
 
@@ -303,7 +304,7 @@ class LSPluginCaller extends EventEmitter {
 
           this._call = async (...args: any) => {
             // parent all will get message before handshake
-            await refChild.call(LSPMSGFn(pl.id), {
+            refChild.call(LSPMSGFn(pl.id), {
               type: args[0],
               payload: Object.assign(args[1] || {}, {
                 $$pid: pl.id,

libs/src/LSPlugin.core.ts

@@ -695,7 +695,9 @@ class PluginLocal extends EventEmitter<
         options.url = this._resolveResourceFullUrl(options.url, this._localRoot)
 
         // file:// for native
-        if (!this.isWebPlugin && !options.url.startsWith('file:')) {
+        if (!this.isWebPlugin &&
+          !options.url.startsWith('file:') &&
+          !options.url.startsWith('lsp:')) {
           options.url = 'assets://' + options.url
         }
       }

libs/src/LSPlugin.ts

@@ -315,7 +315,7 @@ export type ExternalCommandType =
   | 'logseq.ui/toggle-theme'
   | 'logseq.ui/toggle-wide-mode'
 
-export type UserProxyTags = 'app' | 'editor' | 'db' | 'git' | 'ui' | 'assets' | 'utils'
+export type UserProxyNSTags = 'app' | 'editor' | 'db' | 'git' | 'ui' | 'assets' | 'utils'
 
 export type SearchIndiceInitStatus = boolean
 export type SearchBlockItem = {

libs/src/LSPlugin.user.ts

@@ -30,7 +30,7 @@ import {
   IUserOffHook,
   IGitProxy,
   IUIProxy,
-  UserProxyTags,
+  UserProxyNSTags,
   BlockUUID,
   BlockEntity,
   IDatom,
@@ -734,7 +734,7 @@ export class LSPluginUser
   /**
    * @internal
    */
-  _makeUserProxy(target: any, tag?: UserProxyTags) {
+  _makeUserProxy(target: any, nstag?: UserProxyNSTags) {
     const that = this
     const caller = this.caller
 
@@ -744,13 +744,13 @@ export class LSPluginUser
 
         return function (this: any, ...args: any) {
           if (origMethod) {
-            if (args?.length !== 0) args.concat(tag)
+            if (args?.length !== 0) args.concat(nstag)
             const ret = origMethod.apply(that, args)
             if (ret !== PROXY_CONTINUE) return ret
           }
 
           // Handle hook
-          if (tag) {
+          if (nstag) {
             const hookMatcher = propKey.toString().match(/^(once|off|on)/i)
 
             if (hookMatcher != null) {
@@ -771,7 +771,7 @@ export class LSPluginUser
                 opts = args[2]
               }
 
-              type = `hook:${tag}:${safeSnakeCase(type)}`
+              type = `hook:${nstag}:${safeSnakeCase(type)}`
 
               caller[f](type, handler)
 
@@ -796,13 +796,13 @@ export class LSPluginUser
           let method = propKey as string
 
           // TODO: refactor api call with the explicit tag
-          if ((['git', 'ui', 'assets', 'utils'] as UserProxyTags[]).includes(tag)) {
-            method = tag + '_' + method
+          if ((['git', 'ui', 'assets', 'utils'] as UserProxyNSTags[]).includes(nstag)) {
+            method = nstag + '_' + method
           }
 
           // Call host
           return caller.callAsync(`api:call`, {
-            tag,
+            tag: nstag,
             method,
             args: args,
           })

resources/css/shui.css

@@ -228,6 +228,10 @@ html[data-theme=dark] {
 
 div[data-radix-popper-content-wrapper] {
   @apply !z-[999];
+
+  &:has(> .repos-list) {
+    @apply !z-[var(--ls-z-index-level-5)];
+  }
 }
 
 .ui__dialog-overlay {

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -7,6 +7,7 @@
             ["os" :as os]
             ["path" :as node-path]
             [babashka.cli :as cli]
+            [cljs.pprint :as pprint]
             [clojure.edn :as edn]
             [clojure.set :as set]
             [clojure.string :as string]
@@ -89,7 +90,9 @@
         {:page {:build/journal (date-time-util/date->int two-days-ago)}}
 
         ;; Block property blocks and queries
-        {:page {:block/title "Block Properties"}
+        {:page {:block/title "Block Properties"
+                :build/properties
+                {:logseq.property/description "This page demonstrates all the combinations of property types and single/multiple values that are possible."}}
          :blocks
          [{:block/title "default property block" :build/properties {:default "haha"}}
           {:block/title "default property block" :build/properties {:default-many #{"yee" "haw" "sir"}}}
@@ -108,7 +111,9 @@
           {:block/title "date-many property block" :build/properties {:date-many #{[:build/page {:build/journal today-int}]
                                                                                    [:build/page {:build/journal yesterday-int}]}}}
           {:block/title "datetime property block" :build/properties {:datetime timestamp}}]}
-        {:page {:block/title "Property Queries"}
+        {:page {:block/title "Property Queries"
+                :build/properties
+                {:logseq.property/description "This page demonstrates all property type combinations being queried for a specific value. There should be 2 results for each query, one block and one page."}}
          :blocks
          [(query "(property default \"haha\")")
           (query "(property default-many \"haw\")")
@@ -146,7 +151,9 @@
                                                                               [:build/page {:build/journal yesterday-int}]}}}}
         {:page {:block/title "datetime page" :build/properties {:datetime timestamp}}}
 
-        {:page {:block/title "Has Property Queries"}
+        {:page {:block/title "Has Property Queries"
+                :build/properties
+                {:logseq.property/description "This page demonstrates all property type combinations being queried for having a specific property. There should be 2 results for each query, one block and one page."}}
          :blocks
          [(query "(property default)")
           (query "(property default-many)")
@@ -190,6 +197,8 @@
   "Options spec"
   {:help {:alias :h
           :desc "Print help"}
+   :file {:alias :f
+          :desc "File to save generated sqlite.build EDN"}
    :config {:alias :c
             :coerce edn/read-string
             :desc "EDN map to add to config.edn"}})
@@ -209,7 +218,9 @@
             (fse/removeSync db-path))
         conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
                                                   :classpath (cp/get-classpath)})
-        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx (create-init-data))
+        init-data (create-init-data)
+        _ (when (:file options) (fs/writeFileSync (:file options) (with-out-str (pprint/pprint init-data))))
+        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)
         existing-names (set (map :v (d/datoms @conn :avet :block/title)))
         conflicting-names (set/intersection existing-names (set (keep :block/title init-tx)))]
     (when (seq conflicting-names)

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -118,8 +118,8 @@
                        (map str)
                        (into [;; e.g. block/properties :title
                               "block/properties :"
-                              ;; anything org mode except for org.babashka
-                              "org[^\\.]"
+                              ;; anything org mode except for org.babashka or urls like schema.org
+                              "[^\\.]org[^\\.]"
                               "#+BEGIN_"
                               "#+END_"
                               "pre-block"]))

src/electron/electron/core.cljs

@@ -1,26 +1,26 @@
 (ns electron.core
-  (:require [electron.handler :as handler]
+  (:require ["/electron/utils" :as js-utils]
+            ["electron" :refer [BrowserWindow Menu app protocol ipcMain dialog shell] :as electron]
+            ["electron-deeplink" :refer [Deeplink]]
+            ["os" :as os]
+            ["path" :as node-path]
+            [cljs-bean.core :as bean]
+            [clojure.string :as string]
             [electron.db :as db]
+            [electron.exceptions :as exceptions]
+            [electron.fs-watcher :as fs-watcher]
+            [electron.git :as git]
+            [electron.handler :as handler]
+            [electron.logger :as logger]
+            [electron.server :as server]
             [electron.updater :refer [init-updater] :as updater]
+            [electron.url :refer [logseq-url-handler]]
             [electron.utils :refer [*win mac? linux? dev? get-win-from-sender
                                     decode-protected-assets-schema-path send-to-renderer]
              :as utils]
-            [electron.url :refer [logseq-url-handler]]
-            [electron.logger :as logger]
-            [electron.server :as server]
-            [clojure.string :as string]
-            [promesa.core :as p]
-            [cljs-bean.core :as bean]
-            [electron.fs-watcher :as fs-watcher]
-            ["path" :as node-path]
-            ["os" :as os]
-            ["electron" :refer [BrowserWindow Menu app protocol ipcMain dialog shell] :as electron]
-            ["electron-deeplink" :refer [Deeplink]]
-            [electron.git :as git]
             [electron.window :as win]
-            [electron.exceptions :as exceptions]
-            ["/electron/utils" :as js-utils]
-            [logseq.publishing.export :as publish-export]))
+            [logseq.publishing.export :as publish-export]
+            [promesa.core :as p]))
 
 ;; Keep same as main/frontend.util.url
 (defonce LSP_SCHEME "logseq")
@@ -72,7 +72,7 @@
                  (re-find #"(?i)^/[a-zA-Z]:" path))
              (callback #js {:path path})
 
-             ;; assume winwdows unc path
+             ;; assume windows unc path
              utils/win32?
              (do (logger/debug :resolve-assets-url url)
                  (callback #js {:path (str "//" path)}))
@@ -105,15 +105,15 @@
   (p/let [app-path (. app getAppPath)
           asset-filenames (->> (js->clj asset-filenames) (remove nil?))
           root-dir (or output-path (handler/open-dir-dialog))]
-         (when root-dir
-           (publish-export/create-export
-            html
-            app-path
-            repo-path
-            root-dir
-            {:asset-filenames asset-filenames
-             :log-error-fn logger/error
-             :notification-fn #(send-to-renderer :notification %)}))))
+    (when root-dir
+      (publish-export/create-export
+       html
+       app-path
+       repo-path
+       root-dir
+       {:asset-filenames asset-filenames
+        :log-error-fn logger/error
+        :notification-fn #(send-to-renderer :notification %)}))))
 
 (defn setup-app-manager!
   [^js win]
@@ -250,7 +250,7 @@
          ;; Add React developer tool
          (when-let [^js devtoolsInstaller (and dev? (js/require "electron-devtools-installer"))]
            (-> (.default devtoolsInstaller (.-REACT_DEVELOPER_TOOLS devtoolsInstaller))
-             (.then #(js/console.log "Added Extension:" (.-REACT_DEVELOPER_TOOLS devtoolsInstaller)))))
+               (.then #(js/console.log "Added Extension:" (.-REACT_DEVELOPER_TOOLS devtoolsInstaller)))))
 
          (let [t0 (setup-interceptor! app')
                ^js win (win/create-main-window!)
@@ -281,30 +281,30 @@
 
            ;; main window events
            (.on win "close" (fn [e]
-                                  (git/before-graph-close-hook!)
-                                  (when @*quit-dirty? ;; when not updating
-                                    (.preventDefault e)
-
-                                    (let [windows (win/get-all-windows)
-                                          window @*win
-                                          multiple-windows? (> (count windows) 1)]
-                                      (cond
-                                        (or multiple-windows? (not mac?) @win/*quitting?)
-                                        (when window
-                                          (win/close-handler win handler/close-watcher-when-orphaned! e)
-                                          (reset! *win nil))
-
-                                        (and mac? (not multiple-windows?))
+                              (git/before-graph-close-hook!)
+                              (when @*quit-dirty? ;; when not updating
+                                (.preventDefault e)
+
+                                (let [windows (win/get-all-windows)
+                                      window @*win
+                                      multiple-windows? (> (count windows) 1)]
+                                  (cond
+                                    (or multiple-windows? (not mac?) @win/*quitting?)
+                                    (when window
+                                      (win/close-handler win handler/close-watcher-when-orphaned! e)
+                                      (reset! *win nil))
+
+                                    (and mac? (not multiple-windows?))
                                         ;; Just hiding - don't do any actual closing operation
-                                        (do (.preventDefault ^js/Event e)
-                                            (if (and mac? (.isFullScreen win))
-                                              (do (.once win "leave-full-screen" #(.hide win))
-                                                  (.setFullScreen win false))
-                                              (.hide win)))
-                                        :else
-                                        nil)))))
+                                    (do (.preventDefault ^js/Event e)
+                                        (if (and mac? (.isFullScreen win))
+                                          (do (.once win "leave-full-screen" #(.hide win))
+                                              (.setFullScreen win false))
+                                          (.hide win)))
+                                    :else
+                                    nil)))))
            (.on app' "before-quit" (fn [_e]
-                                    (reset! win/*quitting? true)))
+                                     (reset! win/*quitting? true)))
 
            (.on app' "activate" #(when @*win (.show win)))))))
 

src/main/frontend/common/file/core.cljs

@@ -29,13 +29,36 @@
     :else
     content))
 
+(defn- recur-replace-uuid-in-block-title
+  "Return block-title"
+  [ent max-depth]
+  (let [ref-set (loop [result-refs (:block/refs ent)
+                       current-refs (:block/refs ent)
+                       depth 0]
+                  (if (or (>= depth max-depth) (empty? current-refs))
+                    result-refs
+                    (let [next-refs (mapcat :block/refs current-refs)]
+                      (recur (apply conj result-refs next-refs) next-refs (inc depth)))))]
+    (loop [result (db-content/id-ref->title-ref (:block/title ent) ref-set true)
+           last-result nil
+           depth 0]
+      (if (or (>= depth max-depth)
+              (= last-result result))
+        result
+        (recur (db-content/id-ref->title-ref result ref-set true) result (inc depth))))))
+
 (defn- transform-content
   [repo db {:block/keys [collapsed? format pre-block? title page properties] :as b} level {:keys [heading-to-list?]} context]
-  (let [block-ref-not-saved? (and (seq (:block/_refs (d/entity db (:db/id b))))
+  (let [db-based? (sqlite-util/db-based-graph? repo)
+        block-ref-not-saved? (and (seq (:block/_refs (d/entity db (:db/id b))))
                                   (not (string/includes? title (str (:block/uuid b))))
-                                  (not (sqlite-util/db-based-graph? repo)))
+                                  (not db-based?))
         heading (:heading properties)
         markdown? (= :markdown format)
+        title (if db-based?
+                ;; replace [[uuid]] with block's content
+                (recur-replace-uuid-in-block-title (d/entity db (:db/id b)) 10)
+                title)
         content (or title "")
         page-first-child? (= (:db/id b) (ldb/get-first-child db (:db/id page)))
         pre-block? (or pre-block?
@@ -82,11 +105,10 @@
                                     (string/blank? new-content))
                               ""
                               " ")]
-                    (str prefix sep new-content)))
-        content (if block-ref-not-saved?
-                  (gp-property/insert-property repo format content :id (str (:block/uuid b)))
-                  content)]
-    content))
+                    (str prefix sep new-content)))]
+    (if block-ref-not-saved?
+      (gp-property/insert-property repo format content :id (str (:block/uuid b)))
+      content)))
 
 (defn- tree->file-content-aux
   [repo db tree {:keys [init-level] :as opts} context]

src/main/frontend/common/thread_api.cljc

@@ -0,0 +1,38 @@
+(ns frontend.common.thread-api
+  "Macro for defining thread apis, which is invokeable by other threads"
+  #?(:cljs (:require-macros [frontend.common.thread-api]))
+  #?(:cljs (:require [logseq.db :as ldb]
+                     [promesa.core :as p])))
+
+#?(:cljs
+   (def *thread-apis (volatile! {})))
+
+#_:clj-kondo/ignore
+(defmacro defkeyword [& _args])
+
+(defmacro def-thread-api
+  "Define a api invokeable by other threads.
+  e.g. (def-thread-api :thread-api/a-api [arg1 arg2] body)"
+  [qualified-keyword-name params & body]
+  (assert (= "thread-api" (namespace qualified-keyword-name)) qualified-keyword-name)
+  (assert (vector? params) params)
+  `(vswap! *thread-apis assoc
+           ~qualified-keyword-name
+           (fn ~params ~@body)))
+
+#?(:cljs
+   (defn remote-function
+     "Return a promise whose value is transit-str."
+     [qualified-kw-str args-direct-passthrough? args-transit-str-or-args-array]
+     (let [qkw (keyword qualified-kw-str)]
+       (if-let [f (@*thread-apis qkw)]
+         (let [result (apply f (cond-> args-transit-str-or-args-array
+                                 (not args-direct-passthrough?) ldb/read-transit-str))
+               result-promise
+               (if (fn? result) ;; missionary task is a fn
+                 (js/Promise. result)
+                 result)]
+           (p/chain
+            result-promise
+            ldb/write-transit-str))
+         (throw (ex-info (str "not found thread-api: " qualified-kw-str) {}))))))

src/main/frontend/components/all_pages.cljs

@@ -1,14 +1,12 @@
 (ns frontend.components.all-pages
   "All pages"
   (:require [frontend.components.block :as component-block]
-            [frontend.components.page :as component-page]
             [frontend.components.views :as views]
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
             [frontend.state :as state]
             [logseq.common.config :as common-config]
-            [logseq.shui.ui :as shui]
             [rum.core :as rum]))
 
 (defn- columns

src/main/frontend/components/container.cljs

@@ -700,7 +700,7 @@
                                               [(:db/id (db/get-page page)) :page])]
                      (state/sidebar-add-block! current-repo db-id block-type)))
                  (reset! sidebar-inited? true))))
-           (when (state/mobile?)
+           (when (mobile-util/native-platform?)
              (state/set-state! :mobile/show-tabbar? true))
            state)}
   []
@@ -1071,6 +1071,7 @@
       [:a#download-as-json-v2.hidden]
       [:a#download-as-transit-debug.hidden]
       [:a#download-as-sqlite-db.hidden]
+      [:a#download-as-db-edn.hidden]
       [:a#download-as-roam-json.hidden]
       [:a#download-as-html.hidden]
       [:a#download-as-zip.hidden]

src/main/frontend/components/container.css

@@ -281,7 +281,7 @@
 }
 
 .cp__sidebar-left-layout {
-  @apply fixed top-0 left-0 w-[10px] z-[var(--ls-z-index-level-5)];
+  @apply fixed top-0 left-0 w-[10px] z-[var(--ls-z-index-level-4)];
 
   a {
     @apply opacity-80 hover:opacity-100 text-foreground;
@@ -319,7 +319,7 @@
   }
 
   &.is-open {
-    width: 100%;
+    @apply w-full h-full;
 
     .left-sidebar-inner {
       transform: translate3d(0, 0, 0);
@@ -440,12 +440,16 @@
 }
 
 .cp__sidebar-main-content {
-  width: 100%;
-  max-width: var(--ls-main-content-max-width);
-  flex: 1;
+  @apply w-full max-w-[var(--ls-main-content-max-width)] flex-1;
 
   .page {
-    @apply px-4;
+    @apply px-5 pt-4;
+  }
+
+  @screen sm {
+    .page {
+      @apply px-2 pt-0;
+    }
   }
 }
 

src/main/frontend/components/content.cljs

@@ -19,7 +19,6 @@
             [frontend.handler.property :as property-handler]
             [frontend.handler.property.util :as pu]
             [frontend.modules.shortcut.core :as shortcut]
-            [frontend.persist-db.browser :as db-browser]
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
@@ -354,11 +353,9 @@
                {:key "(Dev) Show block content history"
                 :on-click
                 (fn []
-                  (let [^object worker @db-browser/*worker
-                        token (state/get-auth-id-token)
+                  (let [token (state/get-auth-id-token)
                         graph-uuid (ldb/get-graph-rtc-uuid (db/get-db))]
-                    (p/let [result (.rtc-get-block-content-versions worker token graph-uuid (str block-id))
-                            blocks-versions (ldb/read-transit-str result)]
+                    (p/let [blocks-versions (state/<invoke-db-worker :thread-api/rtc-get-block-content-versions token graph-uuid block-id)]
                       (prn :Dev-show-block-content-history)
                       (doseq [[block-uuid versions] blocks-versions]
                         (prn :block-uuid block-uuid)

src/main/frontend/components/export.cljs

@@ -4,6 +4,7 @@
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
+            [frontend.handler.db-based.export :as db-export-handler]
             [frontend.handler.export :as export]
             [frontend.handler.export.html :as export-html]
             [frontend.handler.export.opml :as export-opml]
@@ -100,6 +101,10 @@
           [:div
            [:a.font-medium {:on-click #(export/export-repo-as-zip! current-repo)}
             (t :export-zip)]])
+        (when db-based?
+          [:div
+           [:a.font-medium {:on-click #(db-export-handler/export-repo-as-db-edn! current-repo)}
+            (t :export-db-edn)]])
         (when db-based?
           [:div
            [:a.font-medium {:on-click #(export/export-repo-as-debug-transit! current-repo)}

src/main/frontend/components/imports.cljs

@@ -10,8 +10,9 @@
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
             [frontend.fs :as fs]
-            [frontend.handler.file-based.import :as file-import-handler]
             [frontend.handler.db-based.editor :as db-editor-handler]
+            [frontend.handler.db-based.import :as db-import-handler]
+            [frontend.handler.file-based.import :as file-import-handler]
             [frontend.handler.import :as import-handler]
             [frontend.handler.notification :as notification]
             [frontend.handler.repo :as repo-handler]
@@ -70,7 +71,7 @@
                           :error))))
 
 (defn- lsq-import-handler
-  [e & {:keys [sqlite? debug-transit? graph-name]}]
+  [e & {:keys [sqlite? debug-transit? graph-name db-edn?]}]
   (let [file      (first (array-seq (.-files (.-target e))))
         file-name (some-> (gobj/get file "name")
                           (string/lower-case))
@@ -91,7 +92,7 @@
             (set! (.-onload reader)
                   (fn []
                     (let [buffer (.-result ^js reader)]
-                      (import-handler/import-from-sqlite-db! buffer graph-name finished-cb)
+                      (db-import-handler/import-from-sqlite-db! buffer graph-name finished-cb)
                       (shui/dialog-close!))))
             (set! (.-onerror reader) (fn [e] (js/console.error e)))
             (set! (.-onabort reader) (fn [e]
@@ -99,7 +100,7 @@
                                        (js/console.error e)))
             (.readAsArrayBuffer reader file))))
 
-      debug-transit?
+      (or debug-transit? db-edn?)
       (let [graph-name (string/trim graph-name)]
         (cond
           (string/blank? graph-name)
@@ -112,7 +113,9 @@
           (do
             (state/set-state! :graph/importing :logseq)
             (let [reader (js/FileReader.)
-                  import-f import-handler/import-from-debug-transit!]
+                  import-f (if db-edn?
+                             db-import-handler/import-from-edn-file!
+                             db-import-handler/import-from-debug-transit!)]
               (set! (.-onload reader)
                     (fn [e]
                       (let [text (.. e -target -result)]
@@ -121,7 +124,9 @@
                          text
                          #(do
                             (state/set-state! :graph/importing nil)
-                            (finished-cb))))))
+                            (finished-cb)
+                            ;; graph input not closing
+                            (shui/dialog-close-all!))))))
               (.readAsText reader file)))))
 
       (or edn? json?)
@@ -378,7 +383,7 @@
                                   (let [tx-reports
                                         (gp-exporter/add-file-to-db-graph conn (:file/path m) (:file/content m) opts)]
                                     (doseq [tx-report tx-reports]
-                                      (db-browser/transact! @db-browser/*worker repo (:tx-data tx-report) (:tx-meta tx-report)))))}
+                                      (db-browser/transact! repo (:tx-data tx-report) (:tx-meta tx-report)))))}
           {:keys [files import-state]} (gp-exporter/export-file-graph repo db-conn config-file *files options)]
     (log/info :import-file-graph {:msg (str "Import finished in " (/ (t/in-millis (t/interval start-time (t/now))) 1000) " seconds")})
     (state/set-state! :graph/importing nil)
@@ -449,7 +454,7 @@
    [importing?])
   [:<>])
 
-(rum/defc importer < rum/reactive
+(rum/defc ^:large-vars/cleanup-todo importer < rum/reactive
   [{:keys [query-params]}]
   (let [support-file-based? (config/local-file-based-graph? (state/get-current-repo))
         importing? (state/sub :graph/importing)]
@@ -497,8 +502,6 @@
              [:span.flex.flex-col
               [[:strong "Debug Transit"]
                [:small "Import debug transit file into a new DB graph"]]]
-             ;; Test form style changes
-             #_[:a.button {:on-click #(import-file-to-db-handler nil {:import-graph-fn js/alert})} "Open"]
              [:input.absolute.hidden
               {:id "import-debug-transit"
                :type "file"
@@ -506,11 +509,24 @@
                             (shui/dialog-open!
                              #(set-graph-name-dialog e {:debug-transit? true})))}]])
 
+          (when (or (util/electron?) util/web-platform?)
+            [:label.action-input.flex.items-center.mx-2.my-2
+             [:span.as-flex-center [:i (svg/logo 28)]]
+             [:span.flex.flex-col
+              [[:strong "EDN to DB graph"]
+               [:small "Import a DB graph's EDN export into a new DB graph"]]]
+             [:input.absolute.hidden
+              {:id "import-db-edn"
+               :type "file"
+               :on-change (fn [e]
+                            (shui/dialog-open!
+                             #(set-graph-name-dialog e {:db-edn? true})))}]])
+
           (when (and (util/electron?) support-file-based?)
             [:label.action-input.flex.items-center.mx-2.my-2
              [:span.as-flex-center [:i (svg/logo 28)]]
              [:span.flex.flex-col
-              [[:strong "EDN / JSON"]
+              [[:strong "EDN / JSON to plain text graph"]
                [:small (t :on-boarding/importing-lsq-desc)]]]
              [:input.absolute.hidden
               {:id "import-lsq"

src/main/frontend/components/page.cljs

@@ -117,7 +117,7 @@
     (let [[hover set-hover!] (rum/use-state false)
           click-handler-fn (fn []
                              (p/let [result (editor-handler/insert-first-page-block-if-not-exists! (:block/uuid page))
-                                     result (when (string? result) (:tx-data (ldb/read-transit-str result)))
+                                     result (:tx-data result)
                                      first-child-id (first (map :block/uuid result))
                                      first-child (when first-child-id (db/entity [:block/uuid first-child-id]))]
                                (when first-child
@@ -1091,11 +1091,10 @@
   (let [[graph set-graph!] (hooks/use-state nil)]
     (hooks/use-effect!
      (fn []
-       (p/let [data-str (.build-graph ^js @state/*db-worker (state/get-current-repo)
-                                      (ldb/write-transit-str (assoc settings
+       (p/let [result (state/state/<invoke-db-worker :thread-api/build-graph (state/get-current-repo)
+                                      (assoc settings
                                                                     :type :global
-                                                                    :theme theme)))
-               result (ldb/read-transit-str data-str)]
+                                                                    :theme theme))]
          (set-graph! result)))
      [theme settings])
     (when graph
@@ -1150,8 +1149,7 @@
         dark? (= (:theme opts) "dark")]
     (hooks/use-effect!
      (fn []
-       (p/let [data-str (.build-graph ^js @state/*db-worker (state/get-current-repo) (ldb/write-transit-str opts))
-               result (ldb/read-transit-str data-str)]
+       (p/let [result (state/state/<invoke-db-worker :thread-api/build-graph (state/get-current-repo) opts)]
          (set-graph! result)))
      [opts])
     (when (seq (:nodes graph))

src/main/frontend/components/repo.cljs

@@ -241,7 +241,9 @@
                                                        (not (and rtc-graph? remote?)))
                                                 (state/pub-event! [:graph/open-new-window url])
                                                 (cond
-                                                  (:root graph) ; exists locally
+                                                  ;; exists locally?
+                                                  (or (:root graph)
+                                                      (and db-only? (not rtc-graph?)))
                                                   (state/pub-event! [:graph/switch url])
 
                                                   (and rtc-graph? remote?)

src/main/frontend/components/views.cljs

@@ -283,53 +283,52 @@
                                (and (not (db-property/many? p))
                                     (contains? #{:default :number :checkbox :url :node :date}
                                                (:logseq.property/type p)))))) columns)]
-    (when (or table? (seq columns'))
-      (shui/dropdown-menu
-       (shui/dropdown-menu-trigger
-        {:asChild true}
-        (shui/button
-         {:variant "ghost"
-          :class "text-muted-foreground !px-1"
-          :size :sm}
-         (ui/icon "dots" {:size 15})))
-       (shui/dropdown-menu-content
-        {:align "end"}
-        (shui/dropdown-menu-group
-         (when table?
-           (shui/dropdown-menu-sub
-            (shui/dropdown-menu-sub-trigger
-             "Columns visibility")
-            (shui/dropdown-menu-sub-content
-             (for [column (remove #(or (false? (:column-list? %))
-                                       (:disable-hide? %)) columns)]
-               (shui/dropdown-menu-checkbox-item
-                {:key (str (:id column))
-                 :className "capitalize"
-                 :checked (column-visible? column)
-                 :onCheckedChange #(column-toggle-visibility column %)
-                 :onSelect (fn [e] (.preventDefault e))}
-                (:name column))))))
-         (when (seq columns')
-           (shui/dropdown-menu-sub
-            (shui/dropdown-menu-sub-trigger
-             "Group by")
-            (shui/dropdown-menu-sub-content
-             (for [column columns']
-               (shui/dropdown-menu-checkbox-item
-                {:key (str (:id column))
-                 :className "capitalize"
-                 :checked (= (:id column) (:db/ident (:logseq.property.view/group-by-property view-entity)))
-                 :onCheckedChange (fn [result]
-                                    (if result
-                                      (db-property-handler/set-block-property! (:db/id view-entity) :logseq.property.view/group-by-property
-                                                                               (:db/id (db/entity (:id column))))
-                                      (db-property-handler/remove-block-property! (:db/id view-entity) :logseq.property.view/group-by-property)))
-                 :onSelect (fn [e] (.preventDefault e))}
-                (:name column))))))
-         (shui/dropdown-menu-item
-          {:key "export-edn"
-           :on-click #(db-export-handler/export-view-nodes-data rows)}
-          "Export EDN")))))))
+    (shui/dropdown-menu
+     (shui/dropdown-menu-trigger
+      {:asChild true}
+      (shui/button
+       {:variant "ghost"
+        :class "text-muted-foreground !px-1"
+        :size :sm}
+       (ui/icon "dots" {:size 15})))
+     (shui/dropdown-menu-content
+      {:align "end"}
+      (shui/dropdown-menu-group
+       (when table?
+         (shui/dropdown-menu-sub
+          (shui/dropdown-menu-sub-trigger
+           "Columns visibility")
+          (shui/dropdown-menu-sub-content
+           (for [column (remove #(or (false? (:column-list? %))
+                                     (:disable-hide? %)) columns)]
+             (shui/dropdown-menu-checkbox-item
+              {:key (str (:id column))
+               :className "capitalize"
+               :checked (column-visible? column)
+               :onCheckedChange #(column-toggle-visibility column %)
+               :onSelect (fn [e] (.preventDefault e))}
+              (:name column))))))
+       (when (seq columns')
+         (shui/dropdown-menu-sub
+          (shui/dropdown-menu-sub-trigger
+           "Group by")
+          (shui/dropdown-menu-sub-content
+           (for [column columns']
+             (shui/dropdown-menu-checkbox-item
+              {:key (str (:id column))
+               :className "capitalize"
+               :checked (= (:id column) (:db/ident (:logseq.property.view/group-by-property view-entity)))
+               :onCheckedChange (fn [result]
+                                  (if result
+                                    (db-property-handler/set-block-property! (:db/id view-entity) :logseq.property.view/group-by-property
+                                                                             (:db/id (db/entity (:id column))))
+                                    (db-property-handler/remove-block-property! (:db/id view-entity) :logseq.property.view/group-by-property)))
+               :onSelect (fn [e] (.preventDefault e))}
+              (:name column))))))
+       (shui/dropdown-menu-item
+        {:key "export-edn"
+         :on-click #(db-export-handler/export-view-nodes-data rows)}
+        "Export EDN"))))))
 
 (defn- get-column-size
   [column sized-columns]
@@ -1085,7 +1084,7 @@
                    (let [f (get-in table [:data-fns :add-new-object!])]
                      (f view-entity table)))}
       (ui/icon (if asset? "upload" "plus")))
-     [:div "New record"])))
+     [:div "New node"])))
 
 (rum/defc add-new-row < rum/static
   [view-entity table]
@@ -1672,8 +1671,7 @@
 
 (defn <load-view-data
   [view opts]
-  (p/let [data-str (.get-view-data ^js @state/*db-worker (state/get-current-repo) (:db/id view) (ldb/write-transit-str opts))]
-    (ldb/read-transit-str data-str)))
+  (state/<invoke-db-worker :thread-api/get-view-data (state/get-current-repo) (:db/id view) opts))
 
 (rum/defc view-aux
   [view-entity {:keys [view-parent view-feature-type data query-entity-ids] :as option}]

src/main/frontend/db/async.cljs

@@ -14,7 +14,6 @@
             [frontend.db.react :as react]
             [frontend.db.utils :as db-utils]
             [frontend.handler.file-based.property.util :as property-util]
-            [frontend.persist-db.browser :as db-browser]
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.db :as ldb]
@@ -96,9 +95,8 @@
    Separate from file version because values are lazy loaded"
   [property-id & {:as opts}]
   (when property-id
-    (p/let [data-str (.get-property-values ^js @state/*db-worker (state/get-current-repo)
-                                           (ldb/write-transit-str (assoc opts :property-ident property-id)))]
-      (ldb/read-transit-str data-str))))
+    (state/<invoke-db-worker :thread-api/get-property-values (state/get-current-repo)
+                             (assoc opts :property-ident property-id))))
 
 (defonce *block-cache (atom (cache/lru-cache-factory {} :threshold 1000)))
 (defn <get-block
@@ -133,12 +131,10 @@
       cached-response
 
       :else
-      (when-let [^Object sqlite @db-browser/*worker]
+      (do
         (state/update-state! :db/async-query-loading (fn [s] (conj s name')))
-        (p/let [result-str (.get-blocks sqlite graph
-                                        (ldb/write-transit-str
-                                         [{:id id :opts opts}]))
-                result (ldb/read-transit-str result-str)
+        (p/let [result (state/<invoke-db-worker :thread-api/get-blocks graph
+                                                [{:id id :opts opts}])
                 {:keys [block children] :as result'} (first result)]
           (state/update-state! :db/async-query-loading (fn [s] (disj s name')))
           (if skip-transact?
@@ -160,47 +156,39 @@
 (defn <get-blocks
   [graph ids* & {:as opts}]
   (let [ids (remove (fn [id] (:block.temp/fully-loaded? (db/entity id))) ids*)]
-    (when-let [^Object sqlite @db-browser/*worker]
-      (p/let [result-str (.get-blocks sqlite graph
-                                      (ldb/write-transit-str
-                                       (map (fn [id]
-                                              {:id id :opts opts})
-                                            ids)))
-              result (ldb/read-transit-str result-str)]
-        (let [conn (db/get-db graph false)
-              data (mapcat (fn [{:keys [block children]}] (cons block children)) result)]
-          (d/transact! conn data))))))
+    (p/let [result (state/<invoke-db-worker :thread-api/get-blocks graph
+                                            (map (fn [id]
+                                                   {:id id :opts opts})
+                                                 ids))]
+      (let [conn (db/get-db graph false)
+            data (mapcat (fn [{:keys [block children]}] (cons block children)) result)]
+        (d/transact! conn data)))))
 
 (defn <get-block-parents
   [graph id depth]
   (assert (integer? id))
-  (when-let [^Object worker @db-browser/*worker]
-    (when-let [block-id (:block/uuid (db/entity graph id))]
-      (state/update-state! :db/async-query-loading (fn [s] (conj s (str block-id "-parents"))))
-      (p/let [result-str (.get-block-parents worker graph id depth)
-              result (ldb/read-transit-str result-str)
-              conn (db/get-db graph false)
-              _ (d/transact! conn result)]
-        (state/update-state! :db/async-query-loading (fn [s] (disj s (str block-id "-parents"))))
-        result))))
+  (when-let [block-id (:block/uuid (db/entity graph id))]
+    (state/update-state! :db/async-query-loading (fn [s] (conj s (str block-id "-parents"))))
+    (p/let [result (state/<invoke-db-worker :thread-api/get-block-parents graph id depth)
+            conn (db/get-db graph false)
+            _ (d/transact! conn result)]
+      (state/update-state! :db/async-query-loading (fn [s] (disj s (str block-id "-parents"))))
+      result)))
 
 (defn <get-block-refs
   [graph eid]
   (assert (integer? eid))
-  (when-let [^Object worker @db-browser/*worker]
-    (state/update-state! :db/async-query-loading (fn [s] (conj s (str eid "-refs"))))
-    (p/let [result-str (.get-block-refs worker graph eid)
-            result (ldb/read-transit-str result-str)
-            conn (db/get-db graph false)
-            _ (d/transact! conn result)]
-      (state/update-state! :db/async-query-loading (fn [s] (disj s (str eid "-refs"))))
-      result)))
+  (state/update-state! :db/async-query-loading (fn [s] (conj s (str eid "-refs"))))
+  (p/let [result (state/<invoke-db-worker :thread-api/get-block-refs graph eid)
+          conn (db/get-db graph false)
+          _ (d/transact! conn result)]
+    (state/update-state! :db/async-query-loading (fn [s] (disj s (str eid "-refs"))))
+    result))
 
 (defn <get-block-refs-count
   [graph eid]
   (assert (integer? eid))
-  (when-let [^Object worker @db-browser/*worker]
-    (.get-block-refs-count worker graph eid)))
+  (state/<invoke-db-worker :thread-api/get-block-refs-count graph eid))
 
 (defn <get-all-referenced-blocks-uuid
   "Get all uuids of blocks with any back link exists."

src/main/frontend/db/async/util.cljs

@@ -3,7 +3,6 @@
   (:require [datascript.core :as d]
             [frontend.db.conn :as db-conn]
             [frontend.state :as state]
-            [logseq.db :as ldb]
             [promesa.core :as p]))
 
 (defn <q
@@ -11,50 +10,37 @@
           :or {transact-db? true}
           :as opts} & inputs]
   (assert (not-any? fn? inputs) "Async query inputs can't include fns because fn can't be serialized")
-  (when-let [^Object sqlite @state/*db-worker]
-    (let [*async-queries (:db/async-queries @state/state)
-          async-requested? (get @*async-queries [inputs opts])]
-      (if (and async-requested? transact-db?)
-        (let [db (db-conn/get-db graph)]
-          (apply d/q (first inputs) db (rest inputs)))
-        (p/let [result (.q sqlite graph (ldb/write-transit-str inputs))]
-          (swap! *async-queries assoc [inputs opts] true)
-          (when result
-            (let [result' (ldb/read-transit-str result)]
-              (when (and transact-db? (seq result') (coll? result'))
-                (when-let [conn (db-conn/get-db graph false)]
-                  (let [tx-data (->>
-                                 (if (and (coll? (first result'))
-                                          (not (map? (first result'))))
-                                   (apply concat result')
-                                   result')
-                                 (remove nil?))]
-                    (if (every? map? tx-data)
-                      (try
-                        (d/transact! conn tx-data)
-                        (catch :default e
-                          (js/console.error "<q failed with:" e)
-                          nil))
-                      (js/console.log "<q skipped tx for inputs:" inputs)))))
-              result')))))))
+  (let [*async-queries (:db/async-queries @state/state)
+        async-requested? (get @*async-queries [inputs opts])]
+    (if (and async-requested? transact-db?)
+      (let [db (db-conn/get-db graph)]
+        (apply d/q (first inputs) db (rest inputs)))
+      (p/let [result (state/<invoke-db-worker :thread-api/q graph inputs)]
+        (swap! *async-queries assoc [inputs opts] true)
+        (when result
+          (when (and transact-db? (seq result) (coll? result))
+            (when-let [conn (db-conn/get-db graph false)]
+              (let [tx-data (->>
+                             (if (and (coll? (first result))
+                                      (not (map? (first result))))
+                               (apply concat result)
+                               result)
+                             (remove nil?))]
+                (if (every? map? tx-data)
+                  (try
+                    (d/transact! conn tx-data)
+                    (catch :default e
+                      (js/console.error "<q failed with:" e)
+                      nil))
+                  (js/console.log "<q skipped tx for inputs:" inputs)))))
+          result)))))
 
 (defn <pull
   ([graph id]
    (<pull graph '[*] id))
   ([graph selector id]
-   (when-let [^Object sqlite @state/*db-worker]
-     (p/let [result (.pull sqlite graph (ldb/write-transit-str selector) (ldb/write-transit-str id))]
-       (when result
-         (let [result' (ldb/read-transit-str result)]
-           (when-let [conn (db-conn/get-db graph false)]
-             (d/transact! conn [result']))
-           result'))))))
-
-(comment
-  (defn <pull-many
-    [graph selector ids]
-    (assert (seq ids))
-    (when-let [^Object sqlite @state/*db-worker]
-      (p/let [result (.pull-many sqlite graph (ldb/write-transit-str selector) (ldb/write-transit-str ids))]
-        (when result
-          (ldb/read-transit-str result))))))
+   (p/let [result' (state/<invoke-db-worker :thread-api/pull graph selector id)]
+     (when result'
+       (when-let [conn (db-conn/get-db graph false)]
+         (d/transact! conn [result']))
+       result'))))

src/main/frontend/db/restore.cljs

@@ -1,7 +1,6 @@
 (ns frontend.db.restore
   "Fns for DB restore(from text or sqlite)"
   (:require [cljs-time.core :as t]
-            [datascript.transit :as dt]
             [frontend.db.conn :as db-conn]
             [frontend.persist-db :as persist-db]
             [frontend.state :as state]
@@ -13,9 +12,8 @@
   [repo & {:as opts}]
   (state/set-state! :graph/loading? true)
   (p/let [start-time (t/now)
-          data (persist-db/<fetch-init-data repo opts)
+          {:keys [schema initial-data] :as data} (persist-db/<fetch-init-data repo opts)
           _ (assert (some? data) "No data found when reloading db")
-          {:keys [schema initial-data]} (dt/read-transit-str data)
           conn (try
                  (sqlite-common-db/restore-initial-data initial-data schema)
                  (catch :default e

src/main/frontend/db/rtc/debug_ui.cljs

@@ -5,11 +5,9 @@
             [frontend.db :as db]
             [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.handler.user :as user]
-            [frontend.persist-db.browser :as db-browser]
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
-            [logseq.db :as ldb]
             [logseq.db.frontend.schema :as db-schema]
             [logseq.shui.ui :as shui]
             [missionary.core :as m]
@@ -20,9 +18,9 @@
 
 (defn- stop
   []
-  (let [^object worker @db-browser/*worker]
-    (.rtc-stop worker))
-  (reset! debug-state nil))
+  (p/do!
+   (state/<invoke-db-worker :thread-api/rtc-stop)
+   (reset! debug-state nil)))
 
 (rum/defcs ^:large-vars/cleanup-todo rtc-debug-ui < rum/reactive
   (rum/local nil ::logs)
@@ -60,20 +58,16 @@
       (shui/button
        {:size :sm
         :on-click (fn [_]
-                    (let [^object worker @db-browser/*worker]
-                      (p/let [result (.rtc-get-debug-state worker)
-                              new-state (ldb/read-transit-str result)]
-                        (swap! debug-state (fn [old] (merge old new-state))))))}
+                    (p/let [new-state (state/<invoke-db-worker :thread-api/rtc-get-debug-state)]
+                      (swap! debug-state (fn [old] (merge old new-state)))))}
        (shui/tabler-icon "refresh") "state")
 
       (shui/button
        {:size :sm
         :on-click
         (fn [_]
-          (let [token (state/get-auth-id-token)
-                ^object worker @db-browser/*worker]
-            (p/let [result (.rtc-get-graphs worker token)
-                    graph-list (ldb/read-transit-str result)]
+          (let [token (state/get-auth-id-token)]
+            (p/let [graph-list (state/<invoke-db-worker :thread-api/rtc-get-graphs token)]
               (swap! debug-state assoc
                      :remote-graphs
                      (map
@@ -120,9 +114,8 @@
         {:variant :outline
          :class "text-green-rx-09 border-green-rx-10 hover:text-green-rx-10"
          :on-click (fn []
-                     (let [token (state/get-auth-id-token)
-                           ^object worker @db-browser/*worker]
-                       (.rtc-start worker (state/get-current-repo) token)))}
+                     (let [token (state/get-auth-id-token)]
+                       (state/<invoke-db-worker :thread-api/rtc-start (state/get-current-repo) token)))}
         (shui/tabler-icon "player-play") "start")
 
        [:div.my-2.flex
@@ -132,16 +125,14 @@
                                    ")")
                               {:on-click
                                (fn []
-                                 (let [^object worker @db-browser/*worker]
-                                   (.rtc-toggle-auto-push worker)))})]
+                                 (state/<invoke-db-worker :thread-api/rtc-toggle-auto-push))})]
         [:div.mr-2 (ui/button (str "Toggle remote profile("
                                    (if (:remote-profile? debug-state*)
                                      "ON" "OFF")
                                    ")")
                               {:on-click
                                (fn []
-                                 (let [^object worker @db-browser/*worker]
-                                   (.rtc-toggle-remote-profile worker)))})]
+                                 (state/<invoke-db-worker :thread-api/rtc-toggle-remote-profile))})]
         [:div (shui/button
                {:variant :outline
                 :class "text-red-rx-09 border-red-rx-08 hover:text-red-rx-10"
@@ -159,10 +150,10 @@
                                       user-uuid (some-> (:grant-access-to-user debug-state*) parse-uuid)
                                       user-email (when-not user-uuid (:grant-access-to-user debug-state*))]
                                   (when-let [graph-uuid (:graph-uuid debug-state*)]
-                                    (let [^object worker @db-browser/*worker]
-                                      (.rtc-grant-graph-access worker token graph-uuid
-                                                               (some-> user-uuid vector ldb/write-transit-str)
-                                                               (some-> user-email vector ldb/write-transit-str))))))})
+                                    (state/<invoke-db-worker :thread-api/rtc-grant-graph-access
+                                                             token graph-uuid
+                                                             (some-> user-uuid vector)
+                                                             (some-> user-email vector)))))})
 
         [:b "➡️"]
         [:input.form-input.my-2.py-1
@@ -182,23 +173,23 @@
                               (when-let [graph-name (:download-graph-to-repo debug-state*)]
                                 (when-let [{:keys [graph-uuid graph-schema-version]}
                                            (:graph-uuid-to-download debug-state*)]
-                                  (let [^object worker @db-browser/*worker]
-                                    (prn :download-graph graph-uuid graph-schema-version :to graph-name)
-                                    (p/let [token (state/get-auth-id-token)
-                                            download-info-uuid (.rtc-request-download-graph
-                                                                worker token graph-uuid graph-schema-version)
-                                            download-info-uuid (ldb/read-transit-str download-info-uuid)
-                                            result (.rtc-wait-download-graph-info-ready
-                                                    worker token download-info-uuid graph-uuid graph-schema-version 60000)
-                                            {:keys [_download-info-uuid
-                                                    download-info-s3-url
-                                                    _download-info-tx-instant
-                                                    _download-info-t
-                                                    _download-info-created-at]
-                                             :as result} (ldb/read-transit-str result)]
-                                      (when (not= result :timeout)
-                                        (assert (some? download-info-s3-url) result)
-                                        (.rtc-download-graph-from-s3 worker graph-uuid graph-name download-info-s3-url)))))))})
+                                  (prn :download-graph graph-uuid graph-schema-version :to graph-name)
+                                  (p/let [token (state/get-auth-id-token)
+                                          download-info-uuid (state/<invoke-db-worker
+                                                              :thread-api/rtc-request-download-graph
+                                                              token graph-uuid graph-schema-version)
+                                          {:keys [_download-info-uuid
+                                                  download-info-s3-url
+                                                  _download-info-tx-instant
+                                                  _download-info-t
+                                                  _download-info-created-at]
+                                           :as result}
+                                          (state/<invoke-db-worker :thread-api/rtc-wait-download-graph-info-ready
+                                                                   token download-info-uuid graph-uuid graph-schema-version 60000)]
+                                    (when (not= result :timeout)
+                                      (assert (some? download-info-s3-url) result)
+                                      (state/<invoke-db-worker :thread-api/rtc-download-graph-from-s3
+                                                               graph-uuid graph-name download-info-s3-url))))))})
 
       [:b "➡"]
       [:div.flex.flex-row.items-center.gap-2
@@ -232,9 +223,9 @@
                   :on-click (fn []
                               (let [repo (state/get-current-repo)
                                     token (state/get-auth-id-token)
-                                    remote-graph-name (:upload-as-graph-name debug-state*)
-                                    ^js worker @db-browser/*worker]
-                                (.rtc-async-upload-graph worker repo token remote-graph-name)))})
+                                    remote-graph-name (:upload-as-graph-name debug-state*)]
+                                (state/<invoke-db-worker :thread-api/rtc-async-upload-graph
+                                                         repo token remote-graph-name)))})
       [:b "➡️"]
       [:input.form-input.my-2.py-1.w-32
        {:on-change (fn [e] (swap! debug-state assoc :upload-as-graph-name (util/evalue e)))
@@ -248,10 +239,10 @@
                  {:icon "trash"
                   :on-click (fn []
                               (when-let [{:keys [graph-uuid graph-schema-version]} (:graph-uuid-to-delete debug-state*)]
-                                (let [token (state/get-auth-id-token)
-                                      ^object worker @db-browser/*worker]
+                                (let [token (state/get-auth-id-token)]
                                   (prn ::delete-graph graph-uuid graph-schema-version)
-                                  (.rtc-delete-graph worker token graph-uuid graph-schema-version))))})
+                                  (state/<invoke-db-worker :thread-api/rtc-delete-graph
+                                                           token graph-uuid graph-schema-version))))})
 
       (shui/select
        {:on-value-change (fn [[graph-uuid graph-schema-version]]
@@ -269,6 +260,20 @@
          (for [{:keys [graph-uuid graph-schema-version graph-status]} (:remote-graphs debug-state*)]
            (shui/select-item {:value [graph-uuid graph-schema-version] :disabled (some? graph-status)} graph-uuid)))))]
 
+     [:div.pb-2.flex.flex-row.items-center.gap-2
+      (ui/button "Run server-migrations"
+                 {:on-click (fn []
+                              (let [repo (state/get-current-repo)]
+                                (when-let [server-schema-version (:server-schema-version debug-state*)]
+                                  (state/<invoke-db-worker :thread-api/rtc-add-migration-client-ops
+                                                           repo server-schema-version))))})
+      [:input.form-input.my-2.py-1.w-32
+       {:on-change (fn [e] (swap! debug-state assoc :server-schema-version (util/evalue e)))
+        :on-focus (fn [e] (let [v (.-value (.-target e))]
+                            (when (= v "server migration start version here(e.g. \"64.2\")")
+                              (set! (.-value (.-target e)) ""))))
+        :placeholder "server migration start version here(e.g. \"64.2\")"}]]
+
      [:hr.my-2]
 
      (let [*keys-state (get state ::keys-state)
@@ -278,12 +283,10 @@
          (shui/button
           {:size :sm
            :on-click (fn [_]
-                       (let [^object worker @db-browser/*worker]
-                         (p/let [result1 (.rtc-get-graph-keys worker (state/get-current-repo))
-                                 graph-keys (ldb/read-transit-str result1)
-                                 result2 (some->> (state/get-auth-id-token) (.device-list-devices worker))
-                                 devices (ldb/read-transit-str result2)]
-                           (swap! (get state ::keys-state) #(merge % graph-keys {:devices devices})))))}
+                       (p/let [graph-keys (state/<invoke-db-worker :thread-api/rtc-get-graph-keys (state/get-current-repo))
+                               devices (some->> (state/get-auth-id-token)
+                                                (state/<invoke-db-worker :thread-api/list-devices))]
+                         (swap! (get state ::keys-state) #(merge % graph-keys {:devices devices}))))}
           (shui/tabler-icon "refresh") "keys-state")]
         [:div.pb-4
          [:pre.select-text
@@ -294,10 +297,9 @@
         (shui/button
          {:size :sm
           :on-click (fn [_]
-                      (let [^object worker @db-browser/*worker]
-                        (when-let [device-uuid (not-empty (:remove-device-device-uuid keys-state))]
-                          (when-let [token (state/get-auth-id-token)]
-                            (.device-remove-device worker token device-uuid)))))}
+                      (when-let [device-uuid (not-empty (:remove-device-device-uuid keys-state))]
+                        (when-let [token (state/get-auth-id-token)]
+                          (state/<invoke-db-worker :thread-api/remove-device token device-uuid))))}
          "Remove device:")
         [:input.form-input.my-2.py-1.w-32
          {:on-change (fn [e] (swap! *keys-state assoc :remove-device-device-uuid (util/evalue e)))
@@ -308,11 +310,10 @@
         (shui/button
          {:size :sm
           :on-click (fn [_]
-                      (let [^object worker @db-browser/*worker]
-                        (when-let [device-uuid (not-empty (:remove-public-key-device-uuid keys-state))]
-                          (when-let [key-name (not-empty (:remove-public-key-key-name keys-state))]
-                            (when-let [token (state/get-auth-id-token)]
-                              (.device-remove-device-public-key worker token device-uuid key-name))))))}
+                      (when-let [device-uuid (not-empty (:remove-public-key-device-uuid keys-state))]
+                        (when-let [key-name (not-empty (:remove-public-key-key-name keys-state))]
+                          (when-let [token (state/get-auth-id-token)]
+                            (state/<invoke-db-worker :thread-api/remove-device-public-key token device-uuid key-name)))))}
          "Remove public-key:")
         [:input.form-input.my-2.py-1.w-32
          {:on-change (fn [e] (swap! *keys-state assoc :remove-public-key-device-uuid (util/evalue e)))
@@ -329,11 +330,10 @@
         (shui/button
          {:size :sm
           :on-click (fn [_]
-                      (let [^object worker @db-browser/*worker]
-                        (when-let [token (state/get-auth-id-token)]
-                          (when-let [device-uuid (not-empty (:sync-private-key-device-uuid keys-state))]
-                            (.rtc-sync-current-graph-encrypted-aes-key
-                             worker token (ldb/write-transit-str [(parse-uuid device-uuid)]))))))}
+                      (when-let [token (state/get-auth-id-token)]
+                        (when-let [device-uuid (not-empty (:sync-private-key-device-uuid keys-state))]
+                          (state/<invoke-db-worker :thread-api/rtc-sync-current-graph-encrypted-aes-key
+                                                   token [(parse-uuid device-uuid)]))))}
          "Sync CurrentGraph EncryptedAesKey")
         [:input.form-input.my-2.py-1.w-32
          {:on-change (fn [e] (swap! *keys-state assoc :sync-private-key-device-uuid (util/evalue e)))

src/main/frontend/handler/assets.cljs

@@ -1,6 +1,8 @@
 (ns ^:no-doc frontend.handler.assets
   (:require [cljs-http-missionary.client :as http]
             [clojure.string :as string]
+            [frontend.common.missionary :as c.m]
+            [frontend.common.thread-api :as thread-api :refer [def-thread-api]]
             [frontend.config :as config]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
@@ -8,12 +10,12 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.common.config :as common-config]
-            [frontend.common.missionary :as c.m]
             [logseq.common.path :as path]
             [logseq.common.util :as common-util]
             [medley.core :as medley]
             [missionary.core :as m]
-            [promesa.core :as p]))
+            [promesa.core :as p])
+  (:import [missionary Cancelled]))
 
 (defn alias-enabled?
   []
@@ -301,20 +303,45 @@
     (let [*progress-flow (atom nil)
           http-task (http/get get-url {:with-credentials? false
                                        :response-type :array-buffer
-                                       :*progress-flow *progress-flow})]
-      (c.m/run-task
-       (m/reduce (fn [_ v]
-                   (state/update-state!
-                    :rtc/asset-upload-download-progress
-                    (fn [m] (assoc-in m [repo asset-block-uuid-str] v))))
-                 @*progress-flow)
-       :download-asset-progress
-       :succ (constantly nil))
-      (let [{:keys [status body] :as r} (m/? http-task)]
-        (if-not (http/unexceptional-status? status)
-          {:ex-data {:type :rtc.exception/download-asset-failed :data (dissoc r :body)}}
-          (do (c.m/<? (<write-asset repo asset-block-uuid-str asset-type body))
-              nil))))))
+                                       :*progress-flow *progress-flow})
+          progress-canceler
+          (c.m/run-task
+           (m/reduce (fn [_ v]
+                       (state/update-state!
+                        :rtc/asset-upload-download-progress
+                        (fn [m] (assoc-in m [repo asset-block-uuid-str] v))))
+                     @*progress-flow)
+           :download-asset-progress
+           :succ (constantly nil))]
+      (try
+        (let [{:keys [status body] :as r} (m/? http-task)]
+          (if-not (http/unexceptional-status? status)
+            {:ex-data {:type :rtc.exception/download-asset-failed :data (dissoc r :body)}}
+            (do (c.m/<? (<write-asset repo asset-block-uuid-str asset-type body))
+                nil)))
+        (catch Cancelled e
+          (progress-canceler)
+          (throw e))))))
+
+(def-thread-api :thread-api/unlink-asset
+  [repo asset-block-id asset-type]
+  (<unlink-asset repo asset-block-id asset-type))
+
+(def-thread-api :thread-api/get-all-asset-file-paths
+  [repo]
+  (<get-all-asset-file-paths repo))
+
+(def-thread-api :thread-api/get-asset-file-metadata
+  [repo asset-block-id asset-type]
+  (<get-asset-file-metadata repo asset-block-id asset-type))
+
+(def-thread-api :thread-api/rtc-upload-asset
+  [repo asset-block-uuid-str asset-type checksum put-url]
+  (new-task--rtc-upload-asset repo asset-block-uuid-str asset-type checksum put-url))
+
+(def-thread-api :thread-api/rtc-download-asset
+  [repo asset-block-uuid-str asset-type get-url]
+  (new-task--rtc-download-asset repo asset-block-uuid-str asset-type get-url))
 
 (comment
   ;; read asset

src/main/frontend/handler/common/developer.cljs

@@ -91,8 +91,7 @@
         (notification/show! "No page found" :warning)))))
 
 (defn ^:export validate-db []
-  (when-let [^Object worker @state/*db-worker]
-    (.validate-db worker (state/get-current-repo))))
+  (state/<invoke-db-worker :thread-api/validate-db (state/get-current-repo)))
 
 (defn import-chosen-graph
   [repo]

src/main/frontend/handler/common/page.cljs

@@ -68,10 +68,9 @@
                                 current-user-id
                                 (assoc :created-by current-user-id))
                               options)
-                   result (ui-outliner-tx/transact!
-                           {:outliner-op :create-page}
-                           (outliner-op/create-page! title' options'))
-                   [_page-name page-uuid] (ldb/read-transit-str result)
+                   [_page-name page-uuid] (ui-outliner-tx/transact!
+                                           {:outliner-op :create-page}
+                                           (outliner-op/create-page! title' options'))
                    page (db/get-page (or page-uuid title'))]
              (when redirect?
                (route-handler/redirect-to-page! page-uuid)
@@ -168,9 +167,8 @@
               (notification/show! "Journals enabled" :success)))
            (-> (p/let [res (ui-outliner-tx/transact!
                             {:outliner-op :delete-page}
-                            (outliner-op/delete-page! page-uuid))
-                       res' (ldb/read-transit-str res)]
-                 (if res'
+                            (outliner-op/delete-page! page-uuid))]
+                 (if res
                    (when ok-handler (ok-handler))
                    (when error-handler (error-handler))))
                (p/catch (fn [error]

src/main/frontend/handler/db_based/export.cljs

@@ -1,116 +1,84 @@
 (ns frontend.handler.db-based.export
   "Handles DB graph exports and imports across graphs"
   (:require [cljs.pprint :as pprint]
-            [clojure.edn :as edn]
-            [frontend.db :as db]
+            [clojure.string :as string]
+            [frontend.config :as config]
             [frontend.handler.notification :as notification]
-            [frontend.handler.ui :as ui-handler]
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.util.page :as page-util]
-            [logseq.db :as ldb]
-            [logseq.db.sqlite.export :as sqlite-export]
-            [logseq.shui.ui :as shui]
+            [goog.dom :as gdom]
             [promesa.core :as p]))
 
 (defn ^:export export-block-data []
   ;; Use editor state to locate most recent block
   (if-let [block-uuid (:block-id (first (state/get-editor-args)))]
-    (when-let [^Object worker @state/*db-worker]
-      (p/let [result* (.export-edn worker
-                                   (state/get-current-repo)
-                                   (ldb/write-transit-str {:export-type :block :block-id [:block/uuid block-uuid]}))
-              result (ldb/read-transit-str result*)
-              pull-data (with-out-str (pprint/pprint result))]
-        (.writeText js/navigator.clipboard pull-data)
-        (println pull-data)
-        (notification/show! "Copied block's data!" :success)))
+    (p/let [result (state/<invoke-db-worker :thread-api/export-edn
+                                            (state/get-current-repo)
+                                            {:export-type :block :block-id [:block/uuid block-uuid]})
+            pull-data (with-out-str (pprint/pprint result))]
+      (.writeText js/navigator.clipboard pull-data)
+      (println pull-data)
+      (notification/show! "Copied block's data!" :success))
+
     (notification/show! "No block found" :warning)))
 
 (defn export-view-nodes-data [nodes]
   (let [block-uuids (mapv #(vector :block/uuid (:block/uuid %)) nodes)]
-    (when-let [^Object worker @state/*db-worker]
-      (p/let [result* (.export-edn worker
-                                   (state/get-current-repo)
-                                   (ldb/write-transit-str {:export-type :view-nodes :node-ids block-uuids}))
-              result (ldb/read-transit-str result*)
-              pull-data (with-out-str (pprint/pprint result))]
-        (.writeText js/navigator.clipboard pull-data)
-        (println pull-data)
-        (notification/show! "Copied block's data!" :success)))))
+    (p/let [result (state/<invoke-db-worker :thread-api/export-edn
+                                            (state/get-current-repo)
+                                            {:export-type :view-nodes :node-ids block-uuids})
+            pull-data (with-out-str (pprint/pprint result))]
+      (.writeText js/navigator.clipboard pull-data)
+      (println pull-data)
+      (notification/show! "Copied block's data!" :success))))
 
 (defn ^:export export-page-data []
   (if-let [page-id (page-util/get-current-page-id)]
-    (when-let [^Object worker @state/*db-worker]
-      (p/let [result* (.export-edn worker (state/get-current-repo) (ldb/write-transit-str {:export-type :page :page-id page-id}))
-              result (ldb/read-transit-str result*)
-              pull-data (with-out-str (pprint/pprint result))]
-        (.writeText js/navigator.clipboard pull-data)
-        (println pull-data)
-        (notification/show! "Copied page's data!" :success)))
-    (notification/show! "No page found" :warning)))
-
-(defn ^:export export-graph-ontology-data []
-  (when-let [^Object worker @state/*db-worker]
-    (p/let [result* (.export-edn worker (state/get-current-repo) (ldb/write-transit-str {:export-type :graph-ontology}))
-            result (ldb/read-transit-str result*)
+    (p/let [result (state/<invoke-db-worker :thread-api/export-edn
+                                            (state/get-current-repo)
+                                            {:export-type :page :page-id page-id})
             pull-data (with-out-str (pprint/pprint result))]
       (.writeText js/navigator.clipboard pull-data)
       (println pull-data)
-      (js/console.log (str "Exported " (count (:classes result)) " classes and "
-                           (count (:properties result)) " properties"))
-      (notification/show! "Copied graphs's ontology data!" :success))))
+      (notification/show! "Copied page's data!" :success))
+    (notification/show! "No page found" :warning)))
 
-(defn- import-submit [import-inputs _e]
-  (let [export-map (try (edn/read-string (:import-data @import-inputs)) (catch :default _err ::invalid-import))
-        import-block? (::sqlite-export/block export-map)
-        block (when import-block?
-                (if-let [eid (:block-id (first (state/get-editor-args)))]
-                  (db/entity [:block/uuid eid])
-                  (notification/show! "No block found" :warning)))]
-    (if (= ::invalid-import export-map)
-      (notification/show! "The submitted EDN data is invalid! Fix and try again." :warning)
-      (let [{:keys [init-tx block-props-tx error] :as txs}
-            (try
-              (sqlite-export/build-import export-map
-                                          (db/get-db)
-                                          (when block {:current-block block}))
-              (catch :default e
-                (js/console.error "Import EDN error: " e)
-                {:error "An unexpected error occurred during import. See the javascript console for details."}))]
-        (pprint/pprint txs)
-        (if error
-          (notification/show! error :error)
-          (p/do
-            ;; TODO: Use metadata that supports undo
-            (db/transact! (state/get-current-repo) init-tx
-                          (if import-block? {:save-block true} {::sqlite-export/imported-data? true}))
+(defn ^:export export-graph-ontology-data []
+  (p/let [result (state/<invoke-db-worker :thread-api/export-edn
+                                          (state/get-current-repo)
+                                          {:export-type :graph-ontology})
+          pull-data (with-out-str (pprint/pprint result))]
+    (.writeText js/navigator.clipboard pull-data)
+    (println pull-data)
+    (js/console.log (str "Exported " (count (:classes result)) " classes and "
+                         (count (:properties result)) " properties"))
+    (notification/show! "Copied graphs's ontology data!" :success)))
 
-            (when (seq block-props-tx)
-              (db/transact! (state/get-current-repo) block-props-tx
-                            (if import-block? {:save-block true} {::sqlite-export/imported-data? true})))
+(defn- export-graph-edn-data []
+  (p/let [result (state/<invoke-db-worker :thread-api/export-edn
+                                          (state/get-current-repo)
+                                          {:export-type :graph
+                                           :graph-options {:include-timestamps? true}})
+          pull-data (with-out-str (pprint/pprint result))]
+    pull-data))
 
-            (when-not import-block?
-              (state/clear-async-query-state!)
-              (ui-handler/re-render-root!)
-              (notification/show! "Import successful!" :success))))
-        ;; Also close cmd-k
-        (shui/dialog-close-all!)))))
+;; Copied from handler.export
+(defn- file-name [repo extension]
+  (-> (string/replace repo config/local-db-prefix "")
+      (string/replace #"^/+" "")
+      (str "_" (quot (util/time-ms) 1000))
+      (str "." (string/lower-case (name extension)))))
 
-(defn ^:export import-edn-data
-  []
-  (let [import-inputs (atom {:import-data "" :import-block? false})]
-    (shui/dialog-open!
-     [:div
-      [:label.flex.my-2.text-lg "Import EDN Data"]
-      #_[:label.block.flex.items-center.py-3
-         (shui/checkbox {:on-checked-change #(swap! import-inputs update :import-block? not)})
-         [:small.pl-2 (str "Import into current block")]]
-      (shui/textarea {:placeholder "{}"
-                      :class "overflow-y-auto"
-                      :rows 10
-                      :auto-focus true
-                      :on-change (fn [^js e] (swap! import-inputs assoc :import-data (util/evalue e)))})
-      (shui/button {:class "mt-3"
-                    :on-click (partial import-submit import-inputs)}
-                   "Import")])))
+(defn export-repo-as-db-edn!
+  [repo]
+  (p/let [edn-str (export-graph-edn-data)]
+    (when edn-str
+      (let [data-str (some->> edn-str
+                              js/encodeURIComponent
+                              (str "data:text/edn;charset=utf-8,"))
+            filename (file-name repo :edn)]
+        (when-let [anchor (gdom/getElement "download-as-db-edn")]
+          (.setAttribute anchor "href" data-str)
+          (.setAttribute anchor "download" filename)
+          (.click anchor))))))

src/main/frontend/handler/db_based/import.cljs

@@ -0,0 +1,154 @@
+(ns frontend.handler.db-based.import
+  "Handles DB graph imports"
+  (:require [cljs.pprint :as pprint]
+            [clojure.edn :as edn]
+            [frontend.config :as config]
+            [frontend.db :as db]
+            [frontend.handler.notification :as notification]
+            [frontend.handler.repo :as repo-handler]
+            [frontend.handler.ui :as ui-handler]
+            [frontend.persist-db :as persist-db]
+            [frontend.state :as state]
+            [frontend.util :as util]
+            [logseq.db :as ldb]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.shui.ui :as shui]
+            [promesa.core :as p]))
+
+(defn import-from-sqlite-db!
+  [buffer bare-graph-name finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)]
+    (->
+     (p/do!
+      (persist-db/<import-db graph buffer)
+      (state/add-repo! {:url graph})
+      (repo-handler/restore-and-setup-repo! graph {:import-type :sqlite-db})
+      (state/set-current-repo! graph)
+      (persist-db/<export-db graph {})
+      (db/transact! graph (sqlite-util/import-tx :sqlite-db))
+      (finished-ok-handler))
+     (p/catch
+      (fn [e]
+        (js/console.error e)
+        (notification/show!
+         (str (.-message e))
+         :error))))))
+
+(defn import-from-debug-transit!
+  [bare-graph-name raw finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        datoms (ldb/read-transit-str raw)]
+    (p/do!
+     (persist-db/<new graph {:import-type :debug-transit
+                             :datoms datoms})
+     (state/add-repo! {:url graph})
+     (repo-handler/restore-and-setup-repo! graph {:import-type :debug-transit})
+     (db/transact! graph (sqlite-util/import-tx :debug-transit))
+     (state/set-current-repo! graph)
+     (finished-ok-handler))))
+
+(defn- safe-build-edn-import [export-map import-options]
+  (try
+    (sqlite-export/build-import export-map (db/get-db) import-options)
+    (catch :default e
+      (js/console.error "Import EDN error: " e)
+      {:error "An unexpected error occurred building the import. See the javascript console for details."})))
+
+(defn- import-edn-data-from-file
+  [export-map]
+  (let [{:keys [init-tx block-props-tx misc-tx error] :as _txs} (safe-build-edn-import export-map {})]
+    ;; (cljs.pprint/pprint _txs)
+    (if error
+      (notification/show! error :error)
+      (let [tx-meta {::sqlite-export/imported-data? true}
+            repo (state/get-current-repo)]
+        (p/do
+          (db/transact! repo init-tx tx-meta)
+          (when (seq block-props-tx)
+            (db/transact! repo block-props-tx tx-meta))
+          (when (seq misc-tx)
+            (db/transact! repo misc-tx tx-meta)))))))
+
+(defn import-from-edn-file!
+  "Creates a new DB graph and imports sqlite.build EDN file"
+  [bare-graph-name file-body finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        finished-error-handler
+        #(do
+           (state/set-state! :graph/importing nil)
+           (shui/dialog-close-all!))
+        edn-data (try
+                   (edn/read-string file-body)
+                   (catch :default e
+                     (js/console.error e)
+                     (notification/show! "The given EDN file is not valid EDN. Please fix and try again."
+                                         :error)
+                     (finished-error-handler)
+                     nil))]
+    (when (some? edn-data)
+      (-> (p/do!
+           (persist-db/<new graph {:import-type :edn})
+           (state/add-repo! {:url graph})
+           (repo-handler/restore-and-setup-repo! graph {:import-type :edn})
+           (state/set-current-repo! graph)
+           (import-edn-data-from-file edn-data)
+           (finished-ok-handler))
+          (p/catch
+           (fn [e]
+             (js/console.error e)
+             (notification/show! (str "Unexpected error: " (.-message e))
+                                 :error)
+             (finished-error-handler)))))))
+
+(defn- import-edn-data-from-form [import-inputs _e]
+  (let [export-map (try (edn/read-string (:import-data @import-inputs)) (catch :default _err ::invalid-import))
+        import-block? (::sqlite-export/block export-map)
+        block (when import-block?
+                (if-let [eid (:block-id (first (state/get-editor-args)))]
+                  (db/entity [:block/uuid eid])
+                  (notification/show! "No block found" :warning)))]
+    (if (= ::invalid-import export-map)
+      (notification/show! "The submitted EDN data is invalid! Please fix and try again." :warning)
+      (let [{:keys [init-tx block-props-tx misc-tx error] :as txs}
+            (safe-build-edn-import export-map (when block {:current-block block}))]
+        (pprint/pprint txs)
+        (if error
+          (notification/show! error :error)
+          ;; TODO: When not import-block, use metadata that supports undo
+          (let [tx-meta (if import-block? {:outliner-op :save-block} {::sqlite-export/imported-data? true})
+                repo (state/get-current-repo)]
+            (-> (p/do
+                  (db/transact! repo init-tx tx-meta)
+                  (when (seq block-props-tx)
+                    (db/transact! repo block-props-tx tx-meta))
+                  (when (seq misc-tx)
+                    (db/transact! repo misc-tx tx-meta))
+                  (when-not import-block?
+                    (state/clear-async-query-state!)
+                    (ui-handler/re-render-root!)
+                    (notification/show! "Import successful!" :success)))
+                (p/catch (fn [e]
+                           (js/console.error "Import EDN error: " e)
+                           (notification/show! "An unexpected error occurred during import. See the javascript console for details." :error))))))
+        ;; Also close cmd-k
+        (shui/dialog-close-all!)))))
+
+(defn ^:export import-edn-data-dialog
+  "Displays dialog which allows users to paste and import sqlite.build EDN Data"
+  []
+  (let [import-inputs (atom {:import-data "" :import-block? false})]
+    (shui/dialog-open!
+     [:div
+      [:label.flex.my-2.text-lg "Import EDN Data"]
+      #_[:label.block.flex.items-center.py-3
+         (shui/checkbox {:on-checked-change #(swap! import-inputs update :import-block? not)})
+         [:small.pl-2 (str "Import into current block")]]
+      (shui/textarea {:placeholder "{}"
+                      :class "overflow-y-auto"
+                      :rows 10
+                      :auto-focus true
+                      :on-change (fn [^js e] (swap! import-inputs assoc :import-data (util/evalue e)))})
+      (shui/button {:class "mt-3"
+                    :on-click (partial import-edn-data-from-form import-inputs)}
+                   "Import")])))

src/main/frontend/handler/db_based/property.cljs

@@ -1,21 +1,16 @@
 (ns frontend.handler.db-based.property
   "db based property handler"
-  (:require [frontend.modules.outliner.ui :as ui-outliner-tx]
+  (:require [frontend.db :as db]
             [frontend.modules.outliner.op :as outliner-op]
-            [logseq.outliner.op]
+            [frontend.modules.outliner.ui :as ui-outliner-tx]
             [logseq.db.frontend.property :as db-property]
-            [frontend.db :as db]
-            #_:clj-kondo/ignore
-            [frontend.state :as state]
-            [promesa.core :as p]
-            [logseq.db :as ldb]))
+            [logseq.outliner.op]))
 
 (defn upsert-property!
   [property-id schema property-opts]
-  (p/let [result (ui-outliner-tx/transact!
-                  {:outliner-op :upsert-property}
-                  (outliner-op/upsert-property! property-id schema property-opts))]
-    (ldb/read-transit-str result)))
+  (ui-outliner-tx/transact!
+   {:outliner-op :upsert-property}
+   (outliner-op/upsert-property! property-id schema property-opts)))
 
 (defn set-block-property!
   [block-id property-id value]
@@ -40,37 +35,37 @@
   [block-id property-id property-value]
   (ui-outliner-tx/transact!
    {:outliner-op :delete-property-value}
-    (outliner-op/delete-property-value! block-id property-id property-value)))
+   (outliner-op/delete-property-value! block-id property-id property-value)))
 
 (defn create-property-text-block!
   [block-id property-id value opts]
   (ui-outliner-tx/transact!
    {:outliner-op :create-property-text-block}
-    (outliner-op/create-property-text-block! block-id property-id value opts)))
+   (outliner-op/create-property-text-block! block-id property-id value opts)))
 
 (defn batch-set-property!
   [block-ids property-id value]
   (ui-outliner-tx/transact!
    {:outliner-op :batch-set-property}
-    (outliner-op/batch-set-property! block-ids property-id value)))
+   (outliner-op/batch-set-property! block-ids property-id value)))
 
 (defn batch-remove-property!
   [block-ids property-id]
   (ui-outliner-tx/transact!
    {:outliner-op :batch-remove-property}
-    (outliner-op/batch-remove-property! block-ids property-id)))
+   (outliner-op/batch-remove-property! block-ids property-id)))
 
 (defn class-add-property!
   [class-id property-id]
   (ui-outliner-tx/transact!
    {:outliner-op :class-add-property}
-    (outliner-op/class-add-property! class-id property-id)))
+   (outliner-op/class-add-property! class-id property-id)))
 
 (defn class-remove-property!
   [class-id property-id]
   (ui-outliner-tx/transact!
    {:outliner-op :class-remove-property}
-    (outliner-op/class-remove-property! class-id property-id)))
+   (outliner-op/class-remove-property! class-id property-id)))
 
 (defn batch-set-property-closed-value!
   [block-ids db-ident closed-value]
@@ -91,10 +86,10 @@
   [property-id value]
   (ui-outliner-tx/transact!
    {:outliner-op :delete-closed-value}
-    (outliner-op/delete-closed-value! property-id value)))
+   (outliner-op/delete-closed-value! property-id value)))
 
 (defn add-existing-values-to-closed-values!
   [property-id values]
   (ui-outliner-tx/transact!
    {:outliner-op :add-existing-values-to-closed-values}
-    (outliner-op/add-existing-values-to-closed-values! property-id values)))
+   (outliner-op/add-existing-values-to-closed-values! property-id values)))

src/main/frontend/handler/db_based/rtc.cljs

@@ -16,59 +16,53 @@
 
 (defn <rtc-create-graph!
   [repo]
-  (when-let [^js worker @state/*db-worker]
-    (p/do!
-     (js/Promise. user-handler/task--ensure-id&access-token)
-     (let [token (state/get-auth-id-token)
-           repo-name (sqlite-common-db/sanitize-db-name repo)]
-       (.rtc-async-upload-graph worker repo token repo-name)))))
+  (p/do!
+   (js/Promise. user-handler/task--ensure-id&access-token)
+   (let [token (state/get-auth-id-token)
+         repo-name (sqlite-common-db/sanitize-db-name repo)]
+     (state/<invoke-db-worker :thread-api/rtc-async-upload-graph repo token repo-name))))
 
 (defn <rtc-delete-graph!
   [graph-uuid schema-version]
-  (when-let [^js worker @state/*db-worker]
-    (p/do!
-     (js/Promise. user-handler/task--ensure-id&access-token)
-     (let [token (state/get-auth-id-token)]
-       (.rtc-delete-graph worker token graph-uuid schema-version)))))
+  (p/do!
+   (js/Promise. user-handler/task--ensure-id&access-token)
+   (let [token (state/get-auth-id-token)]
+     (state/<invoke-db-worker :thread-api/rtc-delete-graph token graph-uuid schema-version))))
 
 (defn <rtc-download-graph!
   [graph-name graph-uuid graph-schema-version timeout-ms]
   (assert (some? graph-schema-version))
-  (when-let [^js worker @state/*db-worker]
-    (state/set-state! :rtc/downloading-graph-uuid graph-uuid)
-    (p/let [_ (js/Promise. user-handler/task--ensure-id&access-token)
-            token (state/get-auth-id-token)
-            download-info-uuid* (.rtc-request-download-graph worker token graph-uuid graph-schema-version)
-            download-info-uuid (ldb/read-transit-str download-info-uuid*)
-            result (.rtc-wait-download-graph-info-ready
-                    worker token download-info-uuid graph-uuid graph-schema-version timeout-ms)
-            {:keys [_download-info-uuid
-                    download-info-s3-url
-                    _download-info-tx-instant
-                    _download-info-t
-                    _download-info-created-at]
-             :as result} (ldb/read-transit-str result)]
-      (->
-       (when (not= result :timeout)
-         (assert (some? download-info-s3-url) result)
-         (.rtc-download-graph-from-s3 worker graph-uuid graph-name download-info-s3-url))
-       (p/finally
-         #(state/set-state! :rtc/downloading-graph-uuid nil))))))
+  (state/set-state! :rtc/downloading-graph-uuid graph-uuid)
+  (p/let [_ (js/Promise. user-handler/task--ensure-id&access-token)
+          token (state/get-auth-id-token)
+          download-info-uuid (state/<invoke-db-worker
+                              :thread-api/rtc-request-download-graph token graph-uuid graph-schema-version)
+          {:keys [_download-info-uuid
+                  download-info-s3-url
+                  _download-info-tx-instant
+                  _download-info-t
+                  _download-info-created-at]
+           :as result}
+          (state/<invoke-db-worker :thread-api/rtc-wait-download-graph-info-ready
+                                   token download-info-uuid graph-uuid graph-schema-version timeout-ms)]
+    (->
+     (when (not= result :timeout)
+       (assert (some? download-info-s3-url) result)
+       (state/<invoke-db-worker :thread-api/rtc-download-graph-from-s3 graph-uuid graph-name download-info-s3-url))
+     (p/finally
+       #(state/set-state! :rtc/downloading-graph-uuid nil)))))
 
 (defn <rtc-stop!
   []
-  (when-let [^js worker @state/*db-worker]
-    (.rtc-stop worker)))
+  (state/<invoke-db-worker :thread-api/rtc-stop))
 
 (defn <rtc-branch-graph!
   [repo]
-  (when-let [^js worker @state/*db-worker]
-    (p/let [_ (js/Promise. user-handler/task--ensure-id&access-token)
-            token (state/get-auth-id-token)
-            result (.rtc-async-branch-graph worker repo token)
-            start-ex (ldb/read-transit-str result)]
-      (when-let [ex-data* (:ex-data start-ex)]
-        (throw (ex-info (:ex-message start-ex) ex-data*))))))
+  (p/let [_ (js/Promise. user-handler/task--ensure-id&access-token)
+          token (state/get-auth-id-token)
+          start-ex (state/<invoke-db-worker :thread-api/rtc-async-branch-graph repo token)]
+    (when-let [ex-data* (:ex-data start-ex)]
+      (throw (ex-info (:ex-message start-ex) ex-data*)))))
 
 (defn notification-download-higher-schema-graph!
   [graph-name graph-uuid schema-version]
@@ -99,83 +93,75 @@
 
 (defn <rtc-start!
   [repo & {:keys [stop-before-start?] :or {stop-before-start? true}}]
-  (when-let [^js worker @state/*db-worker]
-    (when-let [graph-uuid (ldb/get-graph-rtc-uuid (db/get-db repo))]
-      (p/do!
-       (js/Promise. user-handler/task--ensure-id&access-token)
-       (when stop-before-start? (<rtc-stop!))
-       (let [token (state/get-auth-id-token)]
-         (p/let [result (.rtc-start worker repo token)
-                 start-ex (ldb/read-transit-str result)
-                 ex-data* (:ex-data start-ex)
-                 _ (case (:type ex-data*)
-                     (:rtc.exception/not-rtc-graph
-                      :rtc.exception/not-found-db-conn)
-                     (notification/show! (:ex-message start-ex) :error)
+  (when-let [graph-uuid (ldb/get-graph-rtc-uuid (db/get-db repo))]
+    (p/do!
+     (js/Promise. user-handler/task--ensure-id&access-token)
+     (when stop-before-start? (<rtc-stop!))
+     (let [token (state/get-auth-id-token)]
+       (p/let [start-ex (state/<invoke-db-worker :thread-api/rtc-start repo token)
+               ex-data* (:ex-data start-ex)
+               _ (case (:type ex-data*)
+                   (:rtc.exception/not-rtc-graph
+                    :rtc.exception/not-found-db-conn)
+                   (notification/show! (:ex-message start-ex) :error)
 
-                     :rtc.exception/major-schema-version-mismatched
-                     (case (:sub-type ex-data*)
-                       :download
-                       (notification-download-higher-schema-graph! repo graph-uuid (:remote ex-data*))
-                       :create-branch
-                       (notification-upload-higher-schema-graph! repo)
+                   :rtc.exception/major-schema-version-mismatched
+                   (case (:sub-type ex-data*)
+                     :download
+                     (notification-download-higher-schema-graph! repo graph-uuid (:remote ex-data*))
+                     :create-branch
+                     (notification-upload-higher-schema-graph! repo)
                         ;; else
-                       (do (log/info :start-ex start-ex)
-                           (notification/show! [:div
-                                                [:div (:ex-message start-ex)]
-                                                [:div (-> ex-data*
-                                                          (select-keys [:app :local :remote])
-                                                          pp/pprint
-                                                          with-out-str)]]
-                                               :error)))
+                     (do (log/info :start-ex start-ex)
+                         (notification/show! [:div
+                                              [:div (:ex-message start-ex)]
+                                              [:div (-> ex-data*
+                                                        (select-keys [:app :local :remote])
+                                                        pp/pprint
+                                                        with-out-str)]]
+                                             :error)))
 
-                     :rtc.exception/lock-failed
-                     (js/setTimeout #(<rtc-start! repo) 1000)
+                   :rtc.exception/lock-failed
+                   (js/setTimeout #(<rtc-start! repo) 1000)
 
                       ;; else
-                     nil)]
-           nil))))))
+                   nil)]
+         nil)))))
 
 (defn <get-remote-graphs
   []
-  (when-let [^js worker @state/*db-worker]
-    (p/let [_ (js/Promise. user-handler/task--ensure-id&access-token)
-            token (state/get-auth-id-token)
-            result (.rtc-get-graphs worker token)
-            graphs (ldb/read-transit-str result)
-            result (->> graphs
-                        (remove (fn [graph] (= (:graph-status graph) "deleting")))
-                        (mapv (fn [graph]
-                                (merge
-                                 (let [url (str config/db-version-prefix (:graph-name graph))]
-                                   {:url url
-                                    :GraphName (:graph-name graph)
-                                    :GraphSchemaVersion (:graph-schema-version graph)
-                                    :GraphUUID (:graph-uuid graph)
-                                    :rtc-graph? true})
-                                 (dissoc graph :graph-uuid :graph-name)))))]
-      (state/set-state! :rtc/graphs result))))
+  (p/let [_ (js/Promise. user-handler/task--ensure-id&access-token)
+          token (state/get-auth-id-token)
+          graphs (state/<invoke-db-worker :thread-api/rtc-get-graphs token)
+          result (->> graphs
+                      (remove (fn [graph] (= (:graph-status graph) "deleting")))
+                      (mapv (fn [graph]
+                              (merge
+                               (let [url (str config/db-version-prefix (:graph-name graph))]
+                                 {:url url
+                                  :GraphName (:graph-name graph)
+                                  :GraphSchemaVersion (:graph-schema-version graph)
+                                  :GraphUUID (:graph-uuid graph)
+                                  :rtc-graph? true})
+                               (dissoc graph :graph-uuid :graph-name)))))]
+    (state/set-state! :rtc/graphs result)))
 
 (defn <rtc-get-users-info
   []
   (when-let [graph-uuid (ldb/get-graph-rtc-uuid (db/get-db))]
-    (when-let [^js worker @state/*db-worker]
-      (p/let [token (state/get-auth-id-token)
-              repo (state/get-current-repo)
-              result (.rtc-get-users-info worker token (str graph-uuid))
-              result (ldb/read-transit-str result)]
-        (state/set-state! :rtc/users-info {repo result})))))
+    (p/let [token (state/get-auth-id-token)
+            repo (state/get-current-repo)
+            result (state/<invoke-db-worker :thread-api/rtc-get-users-info token graph-uuid)]
+      (state/set-state! :rtc/users-info {repo result}))))
 
 (defn <rtc-invite-email
   [graph-uuid email]
-  (when-let [^js worker @state/*db-worker]
-    (let [token (state/get-auth-id-token)]
-      (->
-       (p/do!
-        (.rtc-grant-graph-access worker token (str graph-uuid)
-                                 (ldb/write-transit-str [])
-                                 (ldb/write-transit-str [email]))
-        (notification/show! "Invitation sent!" :success))
-       (p/catch (fn [e]
-                  (notification/show! "Something wrong, please try again." :error)
-                  (js/console.error e)))))))
+  (let [token (state/get-auth-id-token)]
+    (->
+     (p/do!
+      (state/<invoke-db-worker :thread-api/rtc-grant-graph-access
+                               token (str graph-uuid) [] [email])
+      (notification/show! "Invitation sent!" :success))
+     (p/catch (fn [e]
+                (notification/show! "Something wrong, please try again." :error)
+                (js/console.error e))))))

src/main/frontend/handler/dnd.cljs

@@ -1,6 +1,7 @@
 (ns frontend.handler.dnd
   "Provides fns for drag and drop"
-  (:require [frontend.db :as db]
+  (:require [frontend.config :as config]
+            [frontend.db :as db]
             [frontend.handler.block :as block-handler]
             [frontend.handler.editor :as editor-handler]
             [frontend.handler.property :as property-handler]
@@ -8,6 +9,7 @@
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.state :as state]
             [logseq.common.util.block-ref :as block-ref]
+            [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]))
 
 (defn move-blocks
@@ -25,10 +27,10 @@
     (cond
       ;; alt pressed, make a block-ref
       (and alt-key? (= (count blocks) 1))
-      (do
+      (let [->ref (if (config/db-based-graph?) page-ref/->page-ref block-ref/->block-ref)]
         (property-handler/file-persist-block-id! (state/get-current-repo) (:block/uuid first-block))
         (editor-handler/api-insert-new-block!
-         (block-ref/->block-ref (:block/uuid first-block))
+         (->ref (:block/uuid first-block))
          {:block-uuid (:block/uuid target-block)
           :sibling? (not nested?)
           :before? top?}))

src/main/frontend/handler/editor.cljs

@@ -1949,7 +1949,8 @@
                                             size)
       (let [new-meta (merge metadata size)
             image-part (first (string/split full_text #"\{"))
-            new-full-text (str image-part (pr-str new-meta))
+            md-link? (string/starts-with? image-part "![")
+            new-full-text (str (if md-link? image-part (str "![image](" image-part ")")) (pr-str new-meta))
             block (db/entity [:block/uuid block-id])
             value (:block/title block)
             new-value (string/replace value full_text new-full-text)]
@@ -2192,8 +2193,9 @@
                    (outliner-save-block! editing-block)))
               result (transact-blocks!)]
         (state/set-block-op-type! nil)
-        (when-let [result (some-> result (ldb/read-transit-str))]
-          (edit-last-block-after-inserted! result) result)))))
+        (when result
+          (edit-last-block-after-inserted! result)
+          result)))))
 
 (defn- block-tree->blocks
   "keep-uuid? - maintain the existing :uuid in tree vec"
@@ -2317,7 +2319,7 @@
                                                              (assoc opts
                                                                     :sibling? sibling?'
                                                                     :insert-template? true)))]
-                   (when result (edit-last-block-after-inserted! (ldb/read-transit-str result))))
+                   (when result (edit-last-block-after-inserted! result)))
 
                  (catch :default ^js/Error e
                    (notification/show!
@@ -3045,8 +3047,8 @@
         (keydown-backspace-handler false e)
 
         (and (= key "#")
-             (and (> pos 0)
-                  (= "#" (util/nth-safe value (dec pos)))))
+             (> pos 0)
+             (= "#" (util/nth-safe value (dec pos))))
         (state/clear-editor-action!)
 
         (and (contains? (set/difference (set (keys reversed-autopair-map))

src/main/frontend/handler/editor/lifecycle.cljs

@@ -1,14 +1,10 @@
 (ns ^:no-doc frontend.handler.editor.lifecycle
-  (:require [frontend.handler.editor :as editor-handler]
+  (:require [dommy.core :as dom]
+            [frontend.db :as db]
+            [frontend.handler.editor :as editor-handler]
             [frontend.state :as state]
             [frontend.util :as util]
-            [goog.dom :as gdom]
-            [frontend.db :as db]
-            [logseq.db :as ldb]
-            [dommy.core :as dom]
-            ;; [clojure.string :as string]
-            ;; [frontend.handler.block :as block-handler]
-            ))
+            [goog.dom :as gdom]))
 
 (defn did-mount!
   [state]
@@ -34,11 +30,10 @@
 
     ;; skip recording editor info when undo or redo is still running
     (when-not (contains? #{:undo :redo} @(:editor/op @state/state))
-      (let [^js worker @state/*db-worker
-            page-id (:block/uuid (:block/page (db/entity (:db/id (state/get-edit-block)))))
+      (let [page-id (:block/uuid (:block/page (db/entity (:db/id (state/get-edit-block)))))
             repo (state/get-current-repo)]
         (when page-id
-          (.record-editor-info worker repo (str page-id) (ldb/write-transit-str (state/get-editor-info))))))
+          (state/<invoke-db-worker :thread-api/record-editor-info repo (str page-id) (state/get-editor-info)))))
 
     (state/set-state! :editor/op nil))
   state)
@@ -58,7 +53,6 @@
 ;;                                (block-handler/sanity-block-content repo (get new-block :block/format :markdown) (:block/title new-block))))))
 ;;   state)
 
-
 (def lifecycle
   {:did-mount did-mount!
    ;; :will-remount will-remount!

src/main/frontend/handler/events.cljs

@@ -72,7 +72,6 @@
             [frontend.modules.outliner.ui :as ui-outliner-tx]
             [frontend.modules.shortcut.core :as st]
             [frontend.persist-db :as persist-db]
-            [frontend.persist-db.browser :as db-browser]
             [frontend.quick-capture :as quick-capture]
             [frontend.rum :as r]
             [frontend.search :as search]
@@ -84,7 +83,6 @@
             [lambdaisland.glogi :as log]
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
-            [logseq.db :as ldb]
             [logseq.shui.ui :as shui]
             [promesa.core :as p]
             [rum.core :as rum]))
@@ -205,21 +203,19 @@
   (state/set-state! :db/async-queries {})
   (st/refresh!)
   (reset! r/*key->atom {})
-
-  (let [^js sqlite @db-browser/*worker]
-    (p/let [writes-finished? (when sqlite (.file-writes-finished? sqlite (state/get-current-repo)))
-            request-finished? (db-transact/request-finished?)]
-      (if (not writes-finished?) ; TODO: test (:sync-graph/init? @state/state)
-        (do
-          (log/info :graph/switch (cond->
-                                   {:request-finished? request-finished?
-                                    :file-writes-finished? writes-finished?}
-                                    (false? request-finished?)
-                                    (assoc :unfinished-requests? @db-transact/*unfinished-request-ids)))
-          (notification/show!
-           "Please wait seconds until all changes are saved for the current graph."
-           :warning))
-        (graph-switch-on-persisted graph opts)))))
+  (p/let [writes-finished? (state/<invoke-db-worker :thread-api/file-writes-finished? (state/get-current-repo))
+          request-finished? (db-transact/request-finished?)]
+    (if (not writes-finished?) ; TODO: test (:sync-graph/init? @state/state)
+      (do
+        (log/info :graph/switch (cond->
+                                 {:request-finished? request-finished?
+                                  :file-writes-finished? writes-finished?}
+                                  (false? request-finished?)
+                                  (assoc :unfinished-requests? @db-transact/*unfinished-request-ids)))
+        (notification/show!
+         "Please wait seconds until all changes are saved for the current graph."
+         :warning))
+      (graph-switch-on-persisted graph opts))))
 
 (defmethod handle :graph/pull-down-remote-graph [[_ graph dir-name]]
   (if (mobile-util/native-ios?)
@@ -371,9 +367,8 @@
                  :preferred-format (state/get-preferred-format)
                  :journals-directory (config/get-journals-directory)
                  :whiteboards-directory (config/get-whiteboards-directory)
-                 :pages-directory (config/get-pages-directory)}
-        worker ^Object @state/*db-worker]
-    (when worker (.set-context worker (ldb/write-transit-str context)))))
+                 :pages-directory (config/get-pages-directory)}]
+    (state/<invoke-db-worker :thread-api/set-context context)))
 
 ;; Hook on a graph is ready to be shown to the user.
 ;; It's different from :graph/restored, as :graph/restored is for window reloaded
@@ -1023,9 +1018,8 @@
                                      {:outliner-op :insert-blocks}
                                      ;; insert a new block
                                      (let [[_p _ block'] (editor-handler/insert-new-block-aux! {} block "")]
-                                       (turn-type! block')))
-                             result' (ldb/read-transit-str result)]
-                       (when-let [id (:block/uuid (first (:blocks result')))]
+                                       (turn-type! block')))]
+                       (when-let [id (:block/uuid (first (:blocks result)))]
                          (db/entity [:block/uuid id])))
                      (p/do!
                       (turn-type! block)

src/main/frontend/handler/export.cljs

@@ -12,8 +12,6 @@
    [frontend.extensions.zip :as zip]
    [frontend.external.roam-export :as roam-export]
    [frontend.handler.assets :as assets-handler]
-   ;; Loads commands
-   [frontend.handler.db-based.export]
    [frontend.handler.export.common :as export-common-handler]
    [frontend.handler.notification :as notification]
    [frontend.idb :as idb]

src/main/frontend/handler/export/common.cljs

@@ -10,10 +10,8 @@
             [frontend.format.mldoc :as mldoc]
             [frontend.modules.file.core :as outliner-file]
             [frontend.modules.outliner.tree :as outliner-tree]
-            [frontend.persist-db.browser :as db-browser]
             [frontend.state :as state]
             [frontend.util :as util :refer [concatv mapcatv removev]]
-            [logseq.db :as ldb]
             [malli.core :as m]
             [malli.util :as mu]
             [promesa.core :as p]))
@@ -190,20 +188,15 @@
 
 (defn <get-all-pages
   [repo]
-  (when-let [^object worker @db-browser/*worker]
-    (p/let [result (.get-all-pages worker repo)]
-      (ldb/read-transit-str result))))
+  (state/<invoke-db-worker :thread-api/export-get-all-pages repo))
 
 (defn <get-debug-datoms
   [repo]
-  (when-let [^object worker @db-browser/*worker]
-    (.get-debug-datoms worker repo)))
+  (state/<invoke-db-worker :thread-api/export-get-debug-datoms repo))
 
 (defn <get-all-page->content
   [repo]
-  (when-let [^object worker @db-browser/*worker]
-    (p/let [result (.get-all-page->content worker repo)]
-      (ldb/read-transit-str result))))
+  (state/<invoke-db-worker :thread-api/export-get-all-page->content repo))
 
 (defn <get-file-contents
   [repo suffix]

src/main/frontend/handler/history.cljs

@@ -7,9 +7,9 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.util.page :as page-util]
+            [goog.functions :refer [debounce]]
             [logseq.db :as ldb]
-            [promesa.core :as p]
-            [goog.functions :refer [debounce]]))
+            [promesa.core :as p]))
 
 (defn- restore-cursor!
   [{:keys [editor-cursors block-content undo?]}]
@@ -33,7 +33,7 @@
 (defn- restore-cursor-and-state!
   [result]
   (state/set-state! :history/paused? true)
-  (let [{:keys [ui-state-str undo?] :as data} (ldb/read-transit-str result)]
+  (let [{:keys [ui-state-str undo?] :as data} result]
     (if ui-state-str
       (let [{:keys [old-state new-state]} (ldb/read-transit-str ui-state-str)]
         (if undo? (restore-app-state! old-state) (restore-app-state! new-state)))
@@ -58,10 +58,9 @@
               (state/set-state! [:editor/last-replace-ref-content-tx repo] nil)
               (editor/save-current-block!)
               (state/clear-editor-action!)
-              (let [^js worker @state/*db-worker]
-                (reset! *last-request (.undo worker repo current-page-uuid-str))
-                (p/let [result @*last-request]
-                  (restore-cursor-and-state! result)))))))))))
+              (reset! *last-request (state/<invoke-db-worker :thread-api/undo repo current-page-uuid-str))
+              (p/let [result @*last-request]
+                (restore-cursor-and-state! result))))))))))
 (defonce undo! (debounce undo-aux! 20))
 
 (let [*last-request (atom nil)]
@@ -79,8 +78,7 @@
            (when (db-transact/request-finished?)
              (util/stop e)
              (state/clear-editor-action!)
-             (let [^js worker @state/*db-worker]
-               (reset! *last-request (.redo worker repo current-page-uuid-str))
-               (p/let [result @*last-request]
-                 (restore-cursor-and-state! result))))))))))
+             (reset! *last-request (state/<invoke-db-worker :thread-api/redo repo current-page-uuid-str))
+             (p/let [result @*last-request]
+               (restore-cursor-and-state! result)))))))))
 (defonce redo! (debounce redo-aux! 20))

src/main/frontend/handler/import.cljs

@@ -5,7 +5,6 @@
             [clojure.edn :as edn]
             [clojure.string :as string]
             [clojure.walk :as walk]
-            [frontend.config :as config]
             [frontend.db :as db]
             [frontend.db.async :as db-async]
             [frontend.format.block :as block]
@@ -13,12 +12,8 @@
             [frontend.handler.editor :as editor]
             [frontend.handler.notification :as notification]
             [frontend.handler.page :as page-handler]
-            [frontend.handler.repo :as repo-handler]
-            [frontend.persist-db :as persist-db]
             [frontend.state :as state]
             [frontend.util :as util]
-            [logseq.db :as ldb]
-            [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.whiteboard :as gp-whiteboard]
             [medley.core :as medley]
@@ -165,25 +160,6 @@
                            form))]
      (walk/postwalk tree-trans-fn tree-vec))))
 
-(defn import-from-sqlite-db!
-  [buffer bare-graph-name finished-ok-handler]
-  (let [graph (str config/db-version-prefix bare-graph-name)]
-    (->
-     (p/do!
-      (persist-db/<import-db graph buffer)
-      (state/add-repo! {:url graph})
-      (repo-handler/restore-and-setup-repo! graph {:import-type :sqlite-db})
-      (state/set-current-repo! graph)
-      (persist-db/<export-db graph {})
-      (db/transact! graph (sqlite-util/import-tx :sqlite-db))
-      (finished-ok-handler))
-     (p/catch
-      (fn [e]
-        (js/console.error e)
-        (notification/show!
-         (str (.-message e))
-         :error))))))
-
 (defn import-from-edn!
   [raw finished-ok-handler]
   (try
@@ -230,16 +206,3 @@
     (async/go
       (async/<! (import-from-tree! clj-data tree-vec-translate-json))
       (finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated
-
-(defn import-from-debug-transit!
-  [bare-graph-name raw finished-ok-handler]
-  (let [graph (str config/db-version-prefix bare-graph-name)
-        datoms (ldb/read-transit-str raw)]
-    (p/do!
-     (persist-db/<new graph {:import-type :debug-transit
-                             :datoms datoms})
-     (state/add-repo! {:url graph})
-     (repo-handler/restore-and-setup-repo! graph {:import-type :debug-transit})
-     (db/transact! graph (sqlite-util/import-tx :debug-transit))
-     (state/set-current-repo! graph)
-     (finished-ok-handler nil))))

src/main/frontend/handler/page.cljs

@@ -28,7 +28,6 @@
             [frontend.mobile.util :as mobile-util]
             [frontend.modules.outliner.op :as outliner-op]
             [frontend.modules.outliner.ui :as ui-outliner-tx]
-            [frontend.persist-db.browser :as db-browser]
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.util.cursor :as cursor]
@@ -120,24 +119,22 @@
 
 (defn rename!
   [page-uuid-or-old-name new-name & {:as _opts}]
-  (when @db-browser/*worker
-    (p/let [page-uuid (cond
-                        (uuid? page-uuid-or-old-name)
-                        page-uuid-or-old-name
-                        (common-util/uuid-string? page-uuid-or-old-name)
-                        page-uuid-or-old-name
-                        :else
-                        (:block/uuid (db/get-page page-uuid-or-old-name)))
-            result (ui-outliner-tx/transact!
-                    {:outliner-op :rename-page}
-                    (outliner-op/rename-page! page-uuid new-name))
-            result' (ldb/read-transit-str result)]
-      (case (if (string? result') (keyword result') result')
-        :invalid-empty-name
-        (notification/show! "Please use a valid name, empty name is not allowed!" :warning)
-        :rename-page-exists
-        (notification/show! "Another page with the new name exists already" :warning)
-        nil))))
+  (p/let [page-uuid (cond
+                      (uuid? page-uuid-or-old-name)
+                      page-uuid-or-old-name
+                      (common-util/uuid-string? page-uuid-or-old-name)
+                      page-uuid-or-old-name
+                      :else
+                      (:block/uuid (db/get-page page-uuid-or-old-name)))
+          result (ui-outliner-tx/transact!
+                  {:outliner-op :rename-page}
+                  (outliner-op/rename-page! page-uuid new-name))]
+    (case (if (string? result) (keyword result) result)
+      :invalid-empty-name
+      (notification/show! "Please use a valid name, empty name is not allowed!" :warning)
+      :rename-page-exists
+      (notification/show! "Another page with the new name exists already" :warning)
+      nil)))
 
 (defn <reorder-favorites!
   [favorites]

src/main/frontend/handler/worker.cljs

@@ -10,18 +10,18 @@
 
 (defmulti handle identity)
 
-(defmethod handle :write-files [_ ^js worker data]
+(defmethod handle :write-files [_ _worker data]
   (let [{:keys [request-id page-id repo files]} data]
     (->
      (p/let [_ (file-handler/alter-files repo files {})]
-       (.page-file-saved worker request-id page-id))
+       (state/<invoke-db-worker :thread-api/page-file-saved request-id page-id))
      (p/catch (fn [error]
                 (notification/show!
                  [:div
                   [:p "Write file failed, please copy the changes to other editors in case of losing data."]
                   "Error: " (str (.-stack error))]
                  :error)
-                (.page-file-saved worker request-id page-id))))))
+                (state/<invoke-db-worker :thread-api/page-file-saved request-id page-id))))))
 
 (defmethod handle :notification [_ _worker data]
   (apply notification/show! data))

src/main/frontend/mobile/index.css

@@ -1,20 +1,9 @@
 .cp__footer {
-  position: fixed;
-  bottom: 0;
-  left: 0;
-  padding: 10px 20px;
-  background-color: var(--ls-primary-background-color);
-  z-index: 10;
-  display: flex;
-  flex: 0 0 auto;
-  white-space: nowrap;
-  height: 80px;
-  align-items: flex-start;
-  box-shadow: 0 0 10px 0 rgba(0, 0, 0, 0.05);
+  @apply fixed bottom-0 left-0 px-5 py-2.5 z-10 flex flex-auto whitespace-nowrap h-20 items-start
+  bg-[var(--ls-primary-background-color)];
 
   .bottom-action {
-    width: 23px;
-    height: 23px;
+    @apply w-6 h-6;
   }
 
   .ti, .timer {
@@ -22,87 +11,53 @@
   }
 
   .timer {
-    position: absolute;
-    left: 40px;
+    @apply absolute left-10;
   }
 }
 
 .action-bar {
-  position: fixed;
-  bottom: 100px;
-  height: 70px;
-  padding: 6px;
-  border-radius: 10px;
-  background-color: var(--ls-secondary-background-color);
-  overflow-x: overlay;
-  overflow-y: hidden;
-  box-shadow: rgba(0, 0, 0, 0.02) 0px 1px 1px 0px, rgba(27, 31, 35, 0.10) 0px 0px 0px 1px;
-  z-index: 100;
+  @apply fixed bottom-[100px] h-[70px] p-1.5 rounded-md overflow-y-hidden overflow-x-auto
+  shadow-md bg-[var(--ls-secondary-background-color)] z-[100];
 
   .action-bar-commands {
-    position: relative;
-    display: flex;
-    justify-content: space-around;
-    width: 120%;
+    @apply relative flex justify-around w-[120%];
 
 
     .ti, .tie {
-      color: var(--ls-primary-text-color);
-      font-size: 23px;
-      opacity: 50%;
+      @apply text-[var(--ls-primary-text-color)] text-[23px] opacity-50;
     }
 
     .description {
-      color: var(--ls-primary-text-color);
-      font-size: 13px;
-      opacity: 60%;
+      @apply text-[var(--ls-primary-text-color)] text-[13px] opacity-60;
     }
 
     button {
-      padding: 5px 10px
+      @apply py-1 px-2;
     }
   }
 }
 
 #mobile-editor-toolbar {
-  position: fixed;
-  bottom: 0;
-  transition: bottom 260ms;
-  transition-timing-function: ease-out;
-  left: 0;
-  width: 100%;
-  z-index: 9999;
-  display: flex;
-  justify-content: space-between;
+  @apply fixed bottom-0 left-0 w-full z-[9999] flex justify-between;
 
   button {
     @apply flex items-center py-2 px-2;
   }
 
   .submenu {
-    @apply fixed left-0 bottom-0 w-full flex-row justify-evenly items-center z-10 bg-base-2;
-    box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.02);
-    overflow-x: overlay;
-    overflow-y: hidden;
-    height: 40px;
-    display: none;
+    @apply fixed left-0 bottom-0 w-full flex-row justify-evenly items-center z-10 bg-base-2
+    hidden overflow-x-auto overflow-y-hidden h-5 border;
 
     &.show-submenu {
-      display: flex;
+      @apply flex;
     }
   }
 
   .toolbar-commands {
-    justify-content: space-between;
-    display: flex;
-    align-items: center;
-    overflow-x: overlay;
-    overflow-y: hidden;
-    overflow-scrolling: touch;
-    width: 95%;
+    @apply flex justify-between items-center w-[95%] overflow-y-hidden overflow-x-auto;
 
     &::-webkit-scrollbar {
-      height: 4px;
+      @apply h-1;
     }
   }
 
@@ -114,29 +69,23 @@
 
 html.is-native-ipad {
   .cp__footer {
-    height: 55px;
-    right: 0;
-    box-shadow: none;
-    flex: 1;
-    index: 0;
+    @apply h-[55px] right-0 shadow-none flex-1 z-0;
   }
 
   .action-bar {
-    width: 70%;
-    min-width: 550px;
+    @apply w-[70%] min-w-[550px];
 
     .action-bar-commands {
-      width: 100%;
+      @apply w-full;
     }
 
     @media (orientation: landscape) {
-      width: 50%;
+      @apply w-1/2;
     }
   }
 }
 
 html.is-native-iphone {
-
   .action-bar {
     left: 3%;
     right: 3%;

src/main/frontend/modules/outliner/ui.cljc

@@ -16,8 +16,7 @@
          (do ~@body)                    ; nested transact!
          (binding [frontend.modules.outliner.op/*outliner-ops* (transient [])]
            ~@body
-           (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*)
-                 worker# @frontend.state/*db-worker]
+           (let [r# (persistent! frontend.modules.outliner.op/*outliner-ops*)]
             ;;  (js/console.groupCollapsed "ui/transact!")
             ;;  (prn :ops r#)
             ;;  (js/console.trace)
@@ -29,14 +28,14 @@
                                                 r#
                                                 (frontend.state/get-date-formatter)
                                                 ~opts))
-               (when (and worker# (seq r#))
+               (when (seq r#)
                  (let [request-id# (frontend.state/get-worker-next-request-id)
-                       request# #(.apply-outliner-ops ^Object worker# (frontend.state/get-current-repo)
-                                                      (logseq.db/write-transit-str r#)
-                                                      (logseq.db/write-transit-str
-                                                       (assoc ~opts
-                                                              :request-id request-id#
-                                                              :editor-info editor-info#)))
+                       request# #(frontend.state/<invoke-db-worker
+                                  :thread-api/apply-outliner-ops
+                                  (frontend.state/get-current-repo)
+                                  r#
+                                  (assoc ~opts
+                                         :request-id request-id#
+                                         :editor-info editor-info#))
                        response# (frontend.state/add-worker-request! request-id# request#)]
-
                    response#)))))))))

src/main/frontend/modules/shortcut/config.cljs

@@ -627,7 +627,7 @@
 
    :misc/import-edn-data {:binding []
                           :db-graph? true
-                          :fn :frontend.handler.db-based.export/import-edn-data}
+                          :fn :frontend.handler.db-based.import/import-edn-data-dialog}
 
    :dev/validate-db   {:binding []
                        :db-graph? true
@@ -647,8 +647,9 @@
   [keyword-fn]
   (fn []
     (if-let [resolved-fn (some-> (namespace keyword-fn)
-                                 ;; export is reserved word
+                                 ;; handle reserved words
                                  (string/replace-first ".export" ".export$")
+                                 (string/replace-first ".import" ".import$")
                                  find-ns-obj
                                  (aget (munge (name keyword-fn))))]
       (resolved-fn)

src/main/frontend/persist_db/browser.cljs

@@ -4,10 +4,10 @@
    This interface uses clj data format as input."
   (:require ["comlink" :as Comlink]
             [electron.ipc :as ipc]
+            [frontend.common.thread-api :as thread-api]
             [frontend.config :as config]
             [frontend.date :as date]
             [frontend.db.transact :as db-transact]
-            [frontend.handler.assets :as assets-handler]
             [frontend.handler.notification :as notification]
             [frontend.handler.worker :as worker-handler]
             [frontend.persist-db.protocol :as protocol]
@@ -16,8 +16,6 @@
             [logseq.db :as ldb]
             [promesa.core :as p]))
 
-(defonce *worker state/*db-worker)
-
 (defn- ask-persist-permission!
   []
   (p/let [persistent? (.persist js/navigator.storage)]
@@ -26,7 +24,7 @@
       (js/console.warn "OPFS storage may be cleared by the browser under storage pressure."))))
 
 (defn- sync-app-state!
-  [^js worker]
+  []
   (add-watch state/state
              :sync-worker-state
              (fn [_ _ prev current]
@@ -37,7 +35,7 @@
                                  (not= (:config prev) (:config current))
                                  (assoc :config (:config current)))]
                  (when (seq new-state)
-                   (.sync-app-state worker (ldb/write-transit-str new-state)))))))
+                   (state/<invoke-db-worker :thread-api/sync-app-state new-state))))))
 
 (defn get-route-data
   [route-match]
@@ -47,7 +45,7 @@
      :query-params (:query-params route-match)}))
 
 (defn- sync-ui-state!
-  [^js worker]
+  []
   (add-watch state/state
              :sync-ui-state
              (fn [_ _ prev current]
@@ -58,15 +56,13 @@
                        old-state (f prev)
                        new-state (f current)]
                    (when (not= new-state old-state)
-                     (.sync-ui-state worker (state/get-current-repo)
-                                     (ldb/write-transit-str {:old-state old-state
-                                                             :new-state new-state}))))))))
+                     (state/<invoke-db-worker :thread-api/sync-ui-state
+                                              (state/get-current-repo)
+                                              {:old-state old-state :new-state new-state})))))))
 
 (defn transact!
-  [^js worker repo tx-data tx-meta]
-  (let [tx-meta' (ldb/write-transit-str tx-meta)
-        tx-data' (ldb/write-transit-str tx-data)
-        ;; TODO: a better way to share those information with worker, maybe using the state watcher to notify the worker?
+  [repo tx-data tx-meta]
+  (let [;; TODO: a better way to share those information with worker, maybe using the state watcher to notify the worker?
         context {:dev? config/dev?
                  :node-test? util/node-test?
                  :validate-db-options (:dev/validate-db-options (state/get-config))
@@ -79,39 +75,7 @@
                  :journals-directory (config/get-journals-directory)
                  :whiteboards-directory (config/get-whiteboards-directory)
                  :pages-directory (config/get-pages-directory)}]
-    (if worker
-      (.transact worker repo tx-data' tx-meta'
-                 (ldb/write-transit-str context))
-      (notification/show! "Latest change was not saved! Please restart the application." :error))))
-
-(defn- with-write-transit-str
-  [p]
-  (p/chain p ldb/write-transit-str))
-
-(deftype Main []
-  Object
-  (readAsset [_this repo asset-block-id asset-type]
-    (assets-handler/<read-asset repo asset-block-id asset-type))
-  (writeAsset [_this repo asset-block-id asset-type data]
-    (assets-handler/<write-asset repo asset-block-id asset-type data))
-  (unlinkAsset [_this repo asset-block-id asset-type]
-    (assets-handler/<unlink-asset repo asset-block-id asset-type))
-  (get-all-asset-file-paths [_this repo]
-    (with-write-transit-str
-      (assets-handler/<get-all-asset-file-paths repo)))
-  (get-asset-file-metadata [_this repo asset-block-id asset-type]
-    (with-write-transit-str
-      (assets-handler/<get-asset-file-metadata repo asset-block-id asset-type)))
-  (rtc-upload-asset [_this repo asset-block-uuid-str asset-type checksum put-url]
-    (with-write-transit-str
-      (js/Promise.
-       (assets-handler/new-task--rtc-upload-asset repo asset-block-uuid-str asset-type checksum put-url))))
-  (rtc-download-asset [_this repo asset-block-uuid-str asset-type get-url]
-    (with-write-transit-str
-      (js/Promise.
-       (assets-handler/new-task--rtc-download-asset repo asset-block-uuid-str asset-type get-url))))
-  (testFn [_this]
-    (prn :debug :works)))
+    (state/<invoke-db-worker :thread-api/transact repo tx-data tx-meta context)))
 
 (defn start-db-worker!
   []
@@ -120,25 +84,32 @@
                        "js/db-worker.js"
                        "static/js/db-worker.js")
           worker (js/Worker. (str worker-url "?electron=" (util/electron?) "&publishing=" config/publishing?))
-          wrapped-worker (Comlink/wrap worker)
+          wrapped-worker* (Comlink/wrap worker)
+          wrapped-worker (fn [qkw direct-pass-args? & args]
+                           (-> (.remoteInvoke ^js wrapped-worker*
+                                              (str (namespace qkw) "/" (name qkw))
+                                              direct-pass-args?
+                                              (if direct-pass-args?
+                                                (into-array args)
+                                                (ldb/write-transit-str args)))
+                               (p/chain ldb/read-transit-str)))
           t1 (util/time-ms)]
-      (Comlink/expose (Main.) worker)
+      (Comlink/expose #js{"remoteInvoke" thread-api/remote-function} worker)
       (worker-handler/handle-message! worker wrapped-worker)
-      (reset! *worker wrapped-worker)
-      (-> (p/let [_ (.init wrapped-worker config/RTC-WS-URL)
+      (reset! state/*db-worker wrapped-worker)
+      (-> (p/let [_ (state/<invoke-db-worker :thread-api/init config/RTC-WS-URL)
                   _ (js/console.debug (str "debug: init worker spent: " (- (util/time-ms) t1) "ms"))
-                  _ (.sync-app-state wrapped-worker
-                                     (ldb/write-transit-str
-                                      {:git/current-repo (state/get-current-repo)
-                                       :config (:config @state/state)}))
-                  _ (sync-app-state! wrapped-worker)
-                  _ (sync-ui-state! wrapped-worker)
+                  _ (state/<invoke-db-worker :thread-api/sync-app-state
+                                             {:git/current-repo (state/get-current-repo)
+                                              :config (:config @state/state)})
+                  _ (sync-app-state!)
+                  _ (sync-ui-state!)
                   _ (ask-persist-permission!)
                   _ (state/pub-event! [:graph/sync-context])]
             (ldb/register-transact-fn!
              (fn worker-transact!
                [repo tx-data tx-meta]
-               (db-transact/transact (partial transact! wrapped-worker)
+               (db-transact/transact transact!
                                      (if (string? repo) repo (state/get-current-repo))
                                      tx-data
                                      tx-meta)))
@@ -171,56 +142,41 @@
 (defrecord InBrowser []
   protocol/PersistentDB
   (<new [_this repo opts]
-    (when-let [^js sqlite @*worker]
-      (.createOrOpenDB sqlite repo (ldb/write-transit-str opts))))
+    (state/<invoke-db-worker :thread-api/create-or-open-db repo opts))
 
   (<list-db [_this]
-    (when-let [^js sqlite @*worker]
-      (-> (.listDB sqlite)
-          (p/then ldb/read-transit-str)
-          (p/catch sqlite-error-handler))))
+    (-> (state/<invoke-db-worker :thread-api/list-db)
+        (p/catch sqlite-error-handler)))
 
   (<unsafe-delete [_this repo]
-    (when-let [^js sqlite @*worker]
-      (.unsafeUnlinkDB sqlite repo)))
+    (state/<invoke-db-worker :thread-api/unsafe-unlink-db repo))
 
   (<release-access-handles [_this repo]
-    (when-let [^js sqlite @*worker]
-      (.releaseAccessHandles sqlite repo)))
+    (state/<invoke-db-worker :thread-api/release-access-handles repo))
 
   (<fetch-initial-data [_this repo opts]
-    (when-let [^js sqlite @*worker]
-      (-> (p/let [db-exists? (.dbExists sqlite repo)
-                  disk-db-data (when-not db-exists? (ipc/ipc :db-get repo))
-                  _ (when disk-db-data
-                      (.importDb sqlite repo disk-db-data))
-                  _ (.createOrOpenDB sqlite repo (ldb/write-transit-str opts))]
-            (.getInitialData sqlite repo))
-          (p/catch sqlite-error-handler))))
+    (-> (p/let [db-exists? (state/<invoke-db-worker :thread-api/db-exists repo)
+                disk-db-data (when-not db-exists? (ipc/ipc :db-get repo))
+                _ (when disk-db-data
+                    (state/<invoke-db-worker-direct-pass-args :thread-api/import-db repo disk-db-data))
+                _ (state/<invoke-db-worker :thread-api/create-or-open-db repo opts)]
+          (state/<invoke-db-worker :thread-api/get-initial-data repo))
+        (p/catch sqlite-error-handler)))
 
   (<export-db [_this repo opts]
-    (when-let [^js sqlite @*worker]
-      (-> (p/let [data (.exportDB sqlite repo)]
-            (when data
-              (if (:return-data? opts)
-                data
-                (<export-db! repo data))))
-          (p/catch (fn [error]
-                     (prn :debug :save-db-error repo)
-                     (js/console.error error)
-                     (notification/show! [:div (str "SQLiteDB save error: " error)] :error) {})))))
+    (-> (p/let [data (state/<invoke-db-worker :thread-api/export-db repo)]
+          (when data
+            (if (:return-data? opts)
+              data
+              (<export-db! repo data))))
+        (p/catch (fn [error]
+                   (prn :debug :save-db-error repo)
+                   (js/console.error error)
+                   (notification/show! [:div (str "SQLiteDB save error: " error)] :error) {}))))
 
   (<import-db [_this repo data]
-    (when-let [^js sqlite @*worker]
-      (-> (.importDb sqlite repo data)
-          (p/catch (fn [error]
-                     (prn :debug :import-db-error repo)
-                     (js/console.error error)
-                     (notification/show! [:div (str "SQLiteDB import error: " error)] :error) {}))))))
-
-(comment
-  (defn clean-all-dbs!
-    []
-    (when-let [sqlite @*sqlite]
-      (.dangerousRemoveAllDbs sqlite)
-      (state/set-current-repo! nil))))
+    (-> (state/<invoke-db-worker-direct-pass-args :thread-api/import-db repo data)
+        (p/catch (fn [error]
+                   (prn :debug :import-db-error repo)
+                   (js/console.error error)
+                   (notification/show! [:div (str "SQLiteDB import error: " error)] :error) {})))))

src/main/frontend/publishing.cljs

@@ -56,8 +56,7 @@
       (state/set-current-repo! repo)
       (p/let [_ (repo-handler/restore-and-setup-repo! repo)
               _ (let [db-transit-str (unescape-html data)]
-                  (when-let [^js worker @state/*db-worker]
-                    (.resetDB worker repo db-transit-str)))
+                  (state/<invoke-db-worker :thread-api/reset-db repo db-transit-str))
               _ (repo-handler/restore-and-setup-repo! repo)]
         (state/set-db-restoring! false)
         (ui-handler/re-render-root!)))))

src/main/frontend/search.cljs

@@ -14,7 +14,6 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.common.config :as common-config]
-            [logseq.db :as ldb]
             [promesa.core :as p]))
 
 (def fuzzy-search fuzzy/fuzzy-search)
@@ -127,9 +126,8 @@
                       (remove (fn [b] (= (:block/uuid b) (:block/uuid entity))))
                       (map (fn [b] [:block/uuid (:block/uuid b)])))
             result (when (seq eids)
-                     (.get-page-unlinked-refs ^Object @state/*db-worker repo (:db/id entity) (ldb/write-transit-str eids)))
-            result' (when result (ldb/read-transit-str result))]
-      (when result' (d/transact! (db/get-db repo false) result'))
-      (some->> result'
+                     (state/<invoke-db-worker :thread-api/get-page-unlinked-refs repo (:db/id entity) eids))]
+      (when result (d/transact! (db/get-db repo false) result))
+      (some->> result
                db-model/sort-by-order-recursive
                db-utils/group-by-page))))

src/main/frontend/search/browser.cljs

@@ -1,57 +1,41 @@
 (ns frontend.search.browser
   "Browser implementation of search protocol"
-  (:require [cljs-bean.core :as bean]
+  (:require [frontend.config :as config]
+            [frontend.handler.file-based.property.util :as property-util]
             [frontend.search.protocol :as protocol]
-            [promesa.core :as p]
-            [frontend.persist-db.browser :as browser]
             [frontend.state :as state]
-            [frontend.config :as config]
-            [frontend.handler.file-based.property.util :as property-util]
-            [logseq.db :as ldb]))
-
-(defonce *sqlite browser/*worker)
+            [promesa.core :as p]))
 
 (defrecord Browser [repo]
   protocol/Engine
   (query [_this q option]
-    (if-let [^js sqlite @*sqlite]
-      (p/let [result (.search-blocks sqlite (state/get-current-repo) q (bean/->js option))]
-        (ldb/read-transit-str result))
-      (p/resolved nil)))
+    (state/<invoke-db-worker :thread-api/search-blocks (state/get-current-repo) q option))
   (rebuild-pages-indice! [_this]
-    (if-let [^js sqlite @*sqlite]
-      (.search-build-pages-indice sqlite repo)
-      (p/resolved nil)))
+    (state/<invoke-db-worker :thread-api/search-build-pages-indice repo))
   (rebuild-blocks-indice! [this]
-    (if-let [^js sqlite @*sqlite]
-      (p/let [repo (state/get-current-repo)
-              file-based? (config/local-file-based-graph? repo)
-              _ (protocol/truncate-blocks! this)
-              result (.search-build-blocks-indice sqlite repo)
-              blocks (if file-based?
-                       (->> (bean/->clj result)
+    (p/let [repo (state/get-current-repo)
+            file-based? (config/local-file-based-graph? repo)
+            _ (protocol/truncate-blocks! this)
+            result (state/<invoke-db-worker :thread-api/search-build-blocks-indice repo)
+            blocks (if file-based?
+                     (->> result
                             ;; remove built-in properties from content
-                            (map #(update % :content
-                                          (fn [content]
-                                            (property-util/remove-built-in-properties (get % :format :markdown) content))))
-                            bean/->js)
-                       result)
-              _ (when (seq blocks)
-                  (.search-upsert-blocks sqlite repo blocks))])
-      (p/resolved nil)))
+                          (map
+                           #(update % :content
+                                    (fn [content]
+                                      (property-util/remove-built-in-properties (get % :format :markdown) content)))))
+                     result)
+            _ (when (seq blocks)
+                (state/<invoke-db-worker :thread-api/search-upsert-blocks repo blocks))]))
   (transact-blocks! [_this {:keys [blocks-to-remove-set
                                    blocks-to-add]}]
-    (if-let [^js sqlite @*sqlite]
-      (let [repo (state/get-current-repo)]
-        (p/let [_ (when (seq blocks-to-remove-set)
-                    (.search-delete-blocks sqlite repo (bean/->js blocks-to-remove-set)))]
-          (when (seq blocks-to-add)
-            (.search-upsert-blocks sqlite repo (bean/->js blocks-to-add)))))
-      (p/resolved nil)))
+    (let [repo (state/get-current-repo)]
+      (p/let [_ (when (seq blocks-to-remove-set)
+                  (state/<invoke-db-worker :thread-api/search-delete-blocks repo blocks-to-remove-set))]
+        (when (seq blocks-to-add)
+          (state/<invoke-db-worker :thread-api/search-upsert-blocks repo blocks-to-add)))))
   (truncate-blocks! [_this]
-    (if-let [^js sqlite @*sqlite]
-      (.search-truncate-tables sqlite (state/get-current-repo))
-      (p/resolved nil)))
+    (state/<invoke-db-worker :thread-api/search-truncate-tables (state/get-current-repo)))
   (remove-db! [_this]
     ;; Already removed in OPFS
     (p/resolved nil)))

src/main/frontend/state.cljs

@@ -33,6 +33,25 @@
 
 (defonce *db-worker (atom nil))
 
+(defn- <invoke-db-worker*
+  [qkw direct-pass-args? args-list]
+  (let [worker @*db-worker]
+    (when (nil? worker)
+      (prn :<invoke-db-worker-error qkw)
+      (throw (ex-info "db-worker has not been initialized" {})))
+    (apply worker qkw direct-pass-args? args-list)))
+
+(defn <invoke-db-worker
+  "invoke db-worker thread api"
+  [qkw & args]
+  (<invoke-db-worker* qkw false args))
+
+(defn <invoke-db-worker-direct-pass-args
+  "invoke db-worker thread api.
+  But directly pass args to db-worker(won't do transit-write on them)."
+  [qkw & args]
+  (<invoke-db-worker* qkw true args))
+
 ;; Stores main application state
 (defonce ^:large-vars/data-var state
   (let [document-mode? (or (storage/get :document/mode?) false)
@@ -52,7 +71,7 @@
       :nfs/user-granted?                     {}
       :nfs/refreshing?                       nil
       :instrument/disabled?                  (storage/get "instrument-disabled")
-     ;; TODO: how to detect the network reliably?
+      ;; TODO: how to detect the network reliably?
       :network/online?         true
       :indexeddb/support?      true
       :me                      nil
@@ -78,7 +97,7 @@
       :ui/navigation-item-collapsed?         {}
       :ui/recent-pages                       (or (storage/get :ui/recent-pages) {})
 
-     ;; right sidebar
+      ;; right sidebar
       :ui/handbooks-open?                    false
       :ui/help-open?                         false
       :ui/fullscreen?                        false
@@ -140,7 +159,7 @@
       :editor/on-paste?                      (atom false)
       :editor/last-key-code                  (atom nil)
       :ui/global-last-key-code               (atom nil)
-      :editor/block-op-type                  nil             ;; :cut, :copy
+      :editor/block-op-type                  nil ;; :cut, :copy
       :editor/block-refs                     (atom #{})
 
       ;; Stores deleted refed blocks, indexed by repo
@@ -229,7 +248,7 @@
       :plugin/navs-settings?                 true
       :plugin/focused-settings               nil ;; plugin id
 
-     ;; pdf
+      ;; pdf
       :pdf/system-win?                       false
       :pdf/current                           nil
       :pdf/ref-highlight                     nil

src/main/frontend/ui.css

@@ -55,7 +55,7 @@
 .ui__notifications {
   @apply fixed top-12 pointer-events-none w-full;
 
-  z-index: var(--ls-z-index-level-4);
+  z-index: var(--ls-z-index-level-5);
 
   &-content {
     @apply inset-0 flex items-end justify-center px-4 py-2

src/main/frontend/worker/crypt.cljs

@@ -1,6 +1,7 @@
 (ns frontend.worker.crypt
   "Fns to en/decrypt some block attrs"
   (:require [datascript.core :as d]
+            [frontend.common.thread-api :refer [def-thread-api]]
             [frontend.worker.state :as worker-state]
             [promesa.core :as p]))
 
@@ -122,3 +123,7 @@
     (assert (some? conn) repo)
     (let [aes-key-datom (first (d/datoms @conn :avet :aes-key-jwk))]
       {:aes-key-jwk (:v aes-key-datom)})))
+
+(def-thread-api :thread-api/rtc-get-graph-keys
+  [repo]
+  (get-graph-keys-jwk repo))

src/main/frontend/worker/db/migrate.cljs

@@ -971,6 +971,88 @@
             (js/console.error e)
             (throw e)))))))
 
+(defn- build-invalid-tx [entity eid]
+  (cond
+    (and (:db/ident entity) (= "logseq.property.attribute" (namespace (:db/ident entity))))
+    [[:db/retractEntity (:db/id entity)]]
+
+    (and (:logseq.property.history/property entity)
+         (nil? (:logseq.property.history/block entity)))
+    [[:db/retractEntity (:db/id entity)]]
+
+    (and (:db/valueType entity)
+         (not (or (:db/ident entity)
+                  (:db/cardinality entity))))
+    [[:db/retract eid :db/valueType]
+     [:db/retract eid :db/cardinality]]
+
+    (= #{:block/tx-id} (set (keys entity)))
+    [[:db/retractEntity (:db/id entity)]]
+
+    (and (seq (:block/refs entity))
+         (not (or (:block/title entity) (:block/content entity) (:property.value/content entity))))
+    [[:db/retractEntity (:db/id entity)]]
+
+    (:logseq.property.node/type entity)
+    [[:db/retract eid :logseq.property.node/type]
+     [:db/retractEntity :logseq.property.node/type]
+     [:db/add eid :logseq.property.node/display-type (:logseq.property.node/type entity)]]
+
+    (and (:db/cardinality entity) (not (ldb/property? entity)))
+    [[:db/add eid :block/tags :logseq.class/Property]
+     [:db/retract eid :block/tags :logseq.class/Page]]
+
+                                ;; (when-let [schema (:block/schema entity)]
+                                ;;   (or (:cardinality schema) (:classes schema)))
+                                ;; (let [schema (:block/schema entity)]
+                                ;;   [[:db/add eid :block/schema (dissoc schema :cardinality :classes)]])
+
+    (and (:logseq.property.asset/type entity)
+         (or (nil? (:logseq.property.asset/checksum entity))
+             (nil? (:logseq.property.asset/size entity))))
+    [[:db/retractEntity eid]]
+
+                                ;; add missing :db/ident for classes && properties
+    (and (ldb/class? entity) (nil? (:db/ident entity)))
+    [[:db/add (:db/id entity) :db/ident (db-class/create-user-class-ident-from-name (:block/title entity))]]
+
+                                ;; fix blocks missing title
+    (and (:block/parent entity) (nil? (:block/title entity)))
+    [[:db/add (:db/id entity) :block/title ""]]
+
+    (and (ldb/property? entity) (nil? (:db/ident entity)))
+    [[:db/add (:db/id entity) :db/ident (db-property/create-user-property-ident-from-name (:block/title entity))]]
+
+                                ;; remove #Page for classes/properties/journals
+    (and (ldb/internal-page? entity) (or (ldb/class? entity) (ldb/property? entity) (ldb/journal? entity)))
+    [[:db/retract (:db/id entity) :block/tags :logseq.class/Page]]
+
+                                ;; remove file entities
+    (and (:file/path entity)
+         (not (contains? #{"logseq/custom.css" "logseq/config.js"  "logseq/config.edn"} (:file/path entity))))
+    [[:db/retractEntity (:db/id entity)]]
+
+                                ;; remove page-less blocks
+    (and (:block/uuid entity) (nil? (:block/title entity)) (nil? (:block/page entity)))
+    [[:db/retractEntity (:db/id entity)]]
+
+    (:block/properties-order entity)
+    [[:db/retract (:db/id entity) :block/properties-order]]
+
+    (:block/macros entity)
+    [[:db/retract (:db/id entity) :block/macros]]
+
+    (and (seq (:block/tags entity)) (not (every? ldb/class? (:block/tags entity))))
+    (let [tags (remove ldb/class? (:block/tags entity))]
+      (map
+       (fn [tag]
+         {:db/id (:db/id tag)
+          :db/ident (or (:db/ident tag) (db-class/create-user-class-ident-from-name (:block/title entity)))
+          :block/tags :logseq.class/Tag})
+       tags))
+    :else
+    nil))
+
 (defn fix-invalid-data!
   [conn invalid-entity-ids]
   (let [db @conn
@@ -992,78 +1074,7 @@
                                                     [:db/retract (:db/id entity) k]))))))
                                         (into {} entity))
                           eid (:db/id entity)
-                          fix (cond
-                                (and (:db/ident entity) (= "logseq.property.attribute" (namespace (:db/ident entity))))
-                                [[:db/retractEntity (:db/id entity)]]
-
-                                (and (:logseq.property.history/property entity)
-                                     (nil? (:logseq.property.history/block entity)))
-                                [[:db/retractEntity (:db/id entity)]]
-
-                                (and (:db/valueType entity)
-                                     (not (or (:db/ident entity)
-                                              (:db/cardinality entity))))
-                                [[:db/retract eid :db/valueType]
-                                 [:db/retract eid :db/cardinality]]
-
-                                (= #{:block/tx-id} (set (keys entity)))
-                                [[:db/retractEntity (:db/id entity)]]
-
-                                (and (seq (:block/refs entity))
-                                     (not (or (:block/title entity) (:block/content entity) (:property.value/content entity))))
-                                [[:db/retractEntity (:db/id entity)]]
-
-                                (:logseq.property.node/type entity)
-                                [[:db/retract eid :logseq.property.node/type]
-                                 [:db/retractEntity :logseq.property.node/type]
-                                 [:db/add eid :logseq.property.node/display-type (:logseq.property.node/type entity)]]
-
-                                (and (:db/cardinality entity) (not (ldb/property? entity)))
-                                [[:db/add eid :block/tags :logseq.class/Property]
-                                 [:db/retract eid :block/tags :logseq.class/Page]]
-
-                                ;; (when-let [schema (:block/schema entity)]
-                                ;;   (or (:cardinality schema) (:classes schema)))
-                                ;; (let [schema (:block/schema entity)]
-                                ;;   [[:db/add eid :block/schema (dissoc schema :cardinality :classes)]])
-
-                                (and (:logseq.property.asset/type entity)
-                                     (or (nil? (:logseq.property.asset/checksum entity))
-                                         (nil? (:logseq.property.asset/size entity))))
-                                [[:db/retractEntity eid]]
-
-                                ;; add missing :db/ident for classes && properties
-                                (and (ldb/class? entity) (nil? (:db/ident entity)))
-                                [[:db/add (:db/id entity) :db/ident (db-class/create-user-class-ident-from-name (:block/title entity))]]
-
-                                (and (ldb/property? entity) (nil? (:db/ident entity)))
-                                [[:db/add (:db/id entity) :db/ident (db-property/create-user-property-ident-from-name (:block/title entity))]]
-
-                                ;; remove #Page for classes/properties/journals
-                                (and (ldb/internal-page? entity) (or (ldb/class? entity) (ldb/property? entity) (ldb/journal? entity)))
-                                [[:db/retract (:db/id entity) :block/tags :logseq.class/Page]]
-
-                                ;; remove file entities
-                                (and (:file/path entity)
-                                     (not (contains? #{"logseq/custom.css" "logseq/config.js"  "logseq/config.edn"} (:file/path entity))))
-                                [[:db/retractEntity (:db/id entity)]]
-
-                                (:block/properties-order entity)
-                                [[:db/retract (:db/id entity) :block/properties-order]]
-
-                                (:block/macros entity)
-                                [[:db/retract (:db/id entity) :block/macros]]
-
-                                (and (seq (:block/tags entity)) (not (every? ldb/class? (:block/tags entity))))
-                                (let [tags (remove ldb/class? (:block/tags entity))]
-                                  (map
-                                   (fn [tag]
-                                     {:db/id (:db/id tag)
-                                      :db/ident (or (:db/ident tag) (db-class/create-user-class-ident-from-name (:block/title entity)))
-                                      :block/tags :logseq.class/Tag})
-                                   tags))
-                                :else
-                                nil)]
+                          fix (build-invalid-tx entity eid)]
                       (into fix wrong-choice)))
                   invalid-entity-ids)
                  distinct)]

src/main/frontend/worker/db/validate.cljs

@@ -12,7 +12,7 @@
                                         {:msg "Validation errors"
                                          :errors errors}])
         (worker-util/post-message :notification
-                                  [(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy when you interact with them. See the javascript console for more.")
+                                  [(str "Validation detected " (count errors) " invalid block(s). These blocks may be buggy. Attempting to fix invalid blocks. Run validation again to see if they were fixed.")
                                    :warning false]))
 
       (worker-util/post-message :notification

src/main/frontend/worker/db_listener.cljs

@@ -1,7 +1,7 @@
 (ns frontend.worker.db-listener
   "Db listeners for worker-db."
-  (:require [cljs-bean.core :as bean]
-            [datascript.core :as d]
+  (:require [datascript.core :as d]
+            [frontend.common.thread-api :as thread-api]
             [frontend.worker.pipeline :as worker-pipeline]
             [frontend.worker.search :as search]
             [frontend.worker.state :as worker-state]
@@ -30,13 +30,11 @@
 
       (when-not from-disk?
         (p/do!
-         (let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')
-               ^js wo (worker-state/get-worker-object)]
-           (when wo
-             (when (seq blocks-to-remove-set)
-               (.search-delete-blocks wo repo (bean/->js blocks-to-remove-set)))
-             (when (seq blocks-to-add)
-               (.search-upsert-blocks wo repo (bean/->js blocks-to-add))))))))
+         (let [{:keys [blocks-to-remove-set blocks-to-add]} (search/sync-search-indice repo tx-report')]
+           (when (seq blocks-to-remove-set)
+             ((@thread-api/*thread-apis :thread-api/search-delete-blocks) repo blocks-to-remove-set))
+           (when (seq blocks-to-add)
+             ((@thread-api/*thread-apis :thread-api/search-upsert-blocks) repo blocks-to-add))))))
     tx-report'))
 
 (comment

src/main/frontend/worker/db_worker.cljs

@@ -8,20 +8,17 @@
             [clojure.string :as string]
             [datascript.core :as d]
             [datascript.storage :refer [IStorage] :as storage]
-            [frontend.common.file.core :as common-file]
-            [frontend.worker.crypt :as worker-crypt]
+            [frontend.common.thread-api :as thread-api :refer [def-thread-api]]
             [frontend.worker.db-listener :as db-listener]
-            [frontend.worker.db-metadata :as worker-db-metadata]
             [frontend.worker.db.migrate :as db-migrate]
             [frontend.worker.db.validate :as worker-db-validate]
-            [frontend.worker.device :as worker-device]
             [frontend.worker.export :as worker-export]
             [frontend.worker.file :as file]
             [frontend.worker.handler.page :as worker-page]
             [frontend.worker.handler.page.file-based.rename :as file-worker-page-rename]
             [frontend.worker.rtc.asset-db-listener]
             [frontend.worker.rtc.client-op :as client-op]
-            [frontend.worker.rtc.core :as rtc-core]
+            [frontend.worker.rtc.core]
             [frontend.worker.rtc.db-listener]
             [frontend.worker.search :as search]
             [frontend.worker.state :as worker-state] ;; [frontend.worker.undo-redo :as undo-redo]
@@ -42,8 +39,7 @@
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.outliner.op :as outliner-op]
             [me.tonsky.persistent-sorted-set :as set :refer [BTSet]]
-            [promesa.core :as p]
-            [shadow.cljs.modern :refer [defclass]]))
+            [promesa.core :as p]))
 
 (defonce *sqlite worker-state/*sqlite)
 (defonce *sqlite-conns worker-state/*sqlite-conns)
@@ -291,8 +287,7 @@
                                          (set! (.-storage s) (.-storage (:eavt @conn)))
                                          s))]
       (d/reset-conn! conn new-db' {:reset-conn! true})
-      (d/reset-schema! conn (:schema new-db))
-      nil)))
+      (d/reset-schema! conn (:schema new-db)))))
 
 (defn- get-dbs
   [repo]
@@ -346,7 +341,7 @@
                                         (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))]
         (swap! *datascript-conns assoc repo conn)
         (swap! *client-ops-conns assoc repo client-ops-conn)
-        (when (not= client-op/schema-in-db (d/schema @client-ops-conn))
+        (when (and (not @*publishing?) (not= client-op/schema-in-db (d/schema @client-ops-conn)))
           (d/reset-schema! client-ops-conn client-op/schema-in-db))
         (when (and db-based? (not initial-data-exists?) (not datoms))
           (let [config (or config "")
@@ -416,7 +411,7 @@
       (p/all (map (fn [dir]
                     (p/let [graph-name (-> (.-name dir)
                                            (string/replace-first ".logseq-pool-" "")
-                                         ;; TODO: DRY
+                                           ;; TODO: DRY
                                            (string/replace "+3A+" ":")
                                            (string/replace "++" "/"))
                             metadata-file-handle (.getFileHandle dir "metadata.edn" #js {:create true})
@@ -425,6 +420,10 @@
                       {:name graph-name
                        :metadata (edn/read-string metadata)})) db-dirs)))))
 
+(def-thread-api :thread-api/list-db
+  []
+  (<list-all-dbs))
+
 (defn- <db-exists?
   [graph]
   (->
@@ -432,7 +431,7 @@
            _dir-handle (.getDirectoryHandle root (str "." (worker-util/get-pool-name graph)))]
      true)
    (p/catch
-    (fn [_e]                           ; not found
+    (fn [_e]                         ; not found
       false))))
 
 (defn- remove-vfs!
@@ -444,496 +443,303 @@
   [repo]
   (worker-state/get-sqlite-conn repo :search))
 
-(defn- with-write-transit-str
-  [task]
-  (p/chain
-   (js/Promise. task)
-   (fn [result]
-     (let [result (when-not (= result @worker-state/*state) result)]
-       (ldb/write-transit-str result)))))
-
-#_:clj-kondo/ignore
-(defclass DBWorker
-  (extends js/Object)
-
-  (constructor
-   [this]
-   (super))
-
-  Object
-
-  (getVersion
-   [_this]
-   (when-let [sqlite @*sqlite]
-     (.-version sqlite)))
-
-  (init
-   [_this rtc-ws-url]
-   (reset! worker-state/*rtc-ws-url rtc-ws-url)
-   (init-sqlite-module!))
-
-  (storeMetadata
-   [_this repo metadata-str]
-   (worker-db-metadata/<store repo metadata-str))
-
-  (listDB
-   [_this]
-   (p/let [dbs (<list-all-dbs)]
-     (ldb/write-transit-str dbs)))
-
-  (createOrOpenDB
-   [_this repo opts-str]
-   (let [{:keys [close-other-db?] :or {close-other-db? true} :as opts} (ldb/read-transit-str opts-str)]
-     (p/do!
-      (when close-other-db?
-        (close-other-dbs! repo))
-      (create-or-open-db! repo (dissoc opts :close-other-db?)))))
-
-  (getMaxTx
-   [_this repo]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (:max-tx @conn)))
-
-  (q [_this repo inputs-str]
-     "Datascript q"
-     (when-let [conn (worker-state/get-datascript-conn repo)]
-       (let [inputs (ldb/read-transit-str inputs-str)
-             result (apply d/q (first inputs) @conn (rest inputs))]
-         (ldb/write-transit-str result))))
-
-  (pull
-   [_this repo selector-str id-str]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [selector (ldb/read-transit-str selector-str)
-           id (ldb/read-transit-str id-str)
-           eid (if (and (vector? id) (= :block/name (first id)))
-                 (:db/id (ldb/get-page @conn (second id)))
-                 id)
-           result (some->> eid
-                           (d/pull @conn selector)
-                           (sqlite-common-db/with-parent @conn))]
-       (ldb/write-transit-str result))))
-
-  (pull-many
-   [_this repo selector-str ids-str]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [selector (ldb/read-transit-str selector-str)
-           ids (ldb/read-transit-str ids-str)
-           result (d/pull-many @conn selector ids)]
-       (ldb/write-transit-str result))))
-
-  (get-right-sibling
-   [_this repo db-id]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [result (ldb/get-right-sibling (d/entity @conn db-id))]
-       (ldb/write-transit-str result))))
-
-  (get-blocks
-   [_this repo requests-str]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [requests (ldb/read-transit-str requests-str)
-           results (mapv (fn [{:keys [id opts]}]
-                           (let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)]
-                             (-> (sqlite-common-db/get-block-and-children @conn id' opts)
-                                 (assoc :id id)))) requests)]
-       (ldb/write-transit-str results))))
-
-  (get-block-refs
-   [_this repo id]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/write-transit-str (ldb/get-block-refs @conn id))))
-
-  (get-block-refs-count
-   [_this repo id]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/get-block-refs-count @conn id)))
-
-  (get-block-parents
-   [_this repo id depth]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [block-id (:block/uuid (d/entity @conn id))
-           parents (->> (ldb/get-block-parents @conn block-id {:depth (or depth 3)})
-                        (map (fn [b] (d/pull @conn '[*] (:db/id b)))))]
-       (ldb/write-transit-str parents))))
-
-  (set-context
-   [_this context]
-   (let [context (if (string? context)
-                   (ldb/read-transit-str context)
-                   context)]
-     (when context (worker-state/update-context! context))
-     nil))
-
-  (transact
-   [_this repo tx-data tx-meta context]
-   (when repo (worker-state/set-db-latest-tx-time! repo))
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (try
-       (let [tx-data' (if (string? tx-data)
-                        (ldb/read-transit-str tx-data)
-                        tx-data)
-             tx-meta (if (string? tx-meta)
-                       (ldb/read-transit-str tx-meta)
-                       tx-meta)
-             tx-data' (if (contains? #{:insert-blocks} (:outliner-op tx-meta))
-                        (map (fn [m]
-                               (if (and (map? m) (nil? (:block/order m)))
-                                 (assoc m :block/order (db-order/gen-key nil))
-                                 m)) tx-data')
-                        tx-data')
-             context (if (string? context)
-                       (ldb/read-transit-str context)
-                       context)
-             _ (when context (worker-state/set-context! context))
-             tx-meta' (cond-> tx-meta
-                        (and (not (:whiteboard/transact? tx-meta))
-                             (not (:rtc-download-graph? tx-meta))) ; delay writes to the disk
-                        (assoc :skip-store? true)
-
-                        true
-                        (dissoc :insert-blocks?))]
-         (when-not (and (:create-today-journal? tx-meta)
-                        (:today-journal-name tx-meta)
-                        (seq tx-data')
-                        (ldb/get-page @conn (:today-journal-name tx-meta))) ; today journal created already
+(def-thread-api :thread-api/get-version
+  []
+  (when-let [sqlite @*sqlite]
+    (.-version sqlite)))
+
+(def-thread-api :thread-api/init
+  [rtc-ws-url]
+  (reset! worker-state/*rtc-ws-url rtc-ws-url)
+  (init-sqlite-module!))
+
+(def-thread-api :thread-api/create-or-open-db
+  [repo opts]
+  (let [{:keys [close-other-db?] :or {close-other-db? true} :as opts} opts]
+    (p/do!
+     (when close-other-db?
+       (close-other-dbs! repo))
+     (create-or-open-db! repo (dissoc opts :close-other-db?))
+     nil)))
+
+(def-thread-api :thread-api/q
+  [repo inputs]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (apply d/q (first inputs) @conn (rest inputs))))
+
+(def-thread-api :thread-api/pull
+  [repo selector id]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (let [eid (if (and (vector? id) (= :block/name (first id)))
+                (:db/id (ldb/get-page @conn (second id)))
+                id)]
+      (some->> eid
+               (d/pull @conn selector)
+               (sqlite-common-db/with-parent @conn)))))
+
+(def-thread-api :thread-api/get-blocks
+  [repo requests]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (mapv (fn [{:keys [id opts]}]
+            (let [id' (if (and (string? id) (common-util/uuid-string? id)) (uuid id) id)]
+              (-> (sqlite-common-db/get-block-and-children @conn id' opts)
+                  (assoc :id id)))) requests)))
+
+(def-thread-api :thread-api/get-block-refs
+  [repo id]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (ldb/get-block-refs @conn id)))
+
+(def-thread-api :thread-api/get-block-refs-count
+  [repo id]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (ldb/get-block-refs-count @conn id)))
+
+(def-thread-api :thread-api/get-block-parents
+  [repo id depth]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (let [block-id (:block/uuid (d/entity @conn id))]
+      (->> (ldb/get-block-parents @conn block-id {:depth (or depth 3)})
+           (map (fn [b] (d/pull @conn '[*] (:db/id b))))))))
+
+(def-thread-api :thread-api/set-context
+  [context]
+  (when context (worker-state/update-context! context))
+  nil)
+
+(def-thread-api :thread-api/transact
+  [repo tx-data tx-meta context]
+  (when repo (worker-state/set-db-latest-tx-time! repo))
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (try
+      (let [tx-data' (if (contains? #{:insert-blocks} (:outliner-op tx-meta))
+                       (map (fn [m]
+                              (if (and (map? m) (nil? (:block/order m)))
+                                (assoc m :block/order (db-order/gen-key nil))
+                                m)) tx-data)
+                       tx-data)
+            _ (when context (worker-state/set-context! context))
+            tx-meta' (cond-> tx-meta
+                       (and (not (:whiteboard/transact? tx-meta))
+                            (not (:rtc-download-graph? tx-meta))) ; delay writes to the disk
+                       (assoc :skip-store? true)
+
+                       true
+                       (dissoc :insert-blocks?))]
+        (when-not (and (:create-today-journal? tx-meta)
+                       (:today-journal-name tx-meta)
+                       (seq tx-data')
+                       (ldb/get-page @conn (:today-journal-name tx-meta))) ; today journal created already
 
            ;; (prn :debug :transact :tx-data tx-data' :tx-meta tx-meta')
 
-           (worker-util/profile "Worker db transact"
-                                (ldb/transact! conn tx-data' tx-meta')))
-         nil)
-       (catch :default e
-         (prn :debug :error)
-         (let [tx-data (if (string? tx-data)
-                         (ldb/read-transit-str tx-data)
-                         tx-data)]
-           (js/console.error e)
-           (prn :debug :tx-data @conn tx-data))))))
-
-  (getInitialData
-   [_this repo]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/write-transit-str (sqlite-common-db/get-initial-data @conn))))
-
-  (get-page-refs-count
-   [_this repo]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/write-transit-str (sqlite-common-db/get-page->refs-count @conn))))
-
-  (closeDB
-   [_this repo]
-   (close-db! repo))
-
-  (resetDB
-   [_this repo db-transit]
-   (reset-db! repo db-transit))
-
-  (unsafeUnlinkDB
-   [_this repo]
-   (p/let [pool (<get-opfs-pool repo)
-           _ (close-db! repo)
-           result (remove-vfs! pool)]
-     nil))
-
-  (releaseAccessHandles
-   [_this repo]
-   (when-let [^js pool (worker-state/get-opfs-pool repo)]
-     (.releaseAccessHandles pool)))
-
-  (dbExists
-   [_this repo]
-   (<db-exists? repo))
-
-  (exportDB
-   [_this repo]
-   (when-let [^js db (worker-state/get-sqlite-conn repo :db)]
-     (.exec db "PRAGMA wal_checkpoint(2)"))
-   (<export-db-file repo))
-
-  (importDb
-   [this repo data]
-   (when-not (string/blank? repo)
-     (p/let [pool (<get-opfs-pool repo)]
-       (<import-db pool data))))
-
-  ;; Search
-  (search-blocks
-   [this repo q option]
-   (p/let [search-db (get-search-db repo)
-           conn (worker-state/get-datascript-conn repo)
-           result (search/search-blocks repo conn search-db q (bean/->clj option))]
-     (ldb/write-transit-str result)))
-
-  (search-upsert-blocks
-   [this repo blocks]
-   (p/let [db (get-search-db repo)]
-     (search/upsert-blocks! db blocks)
-     nil))
-
-  (search-delete-blocks
-   [this repo ids]
-   (p/let [db (get-search-db repo)]
-     (search/delete-blocks! db ids)
-     nil))
-
-  (search-truncate-tables
-   [this repo]
-   (p/let [db (get-search-db repo)]
-     (search/truncate-table! db)
-     nil))
-
-  (search-build-blocks-indice
-   [this repo]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (search/build-blocks-indice repo @conn)))
-
-  (search-build-pages-indice
-   [this repo]
-   nil)
-
-  (apply-outliner-ops
-   [this repo ops-str opts-str]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (try
-       (worker-util/profile
-        "apply outliner ops"
-        (let [ops (ldb/read-transit-str ops-str)
-              opts (ldb/read-transit-str opts-str)
-              result (outliner-op/apply-ops! repo conn ops (worker-state/get-date-formatter repo) opts)]
-          (ldb/write-transit-str result)))
-       (catch :default e
-         (let [data (ex-data e)
-               {:keys [type payload]} (when (map? data) data)]
-           (case type
-             :notification
-             (worker-util/post-message type [(:message payload) (:type payload)])
-             (throw e)))))))
-
-  (file-writes-finished?
-   [this repo]
-   (let [conn (worker-state/get-datascript-conn repo)
-         writes @file/*writes]
-     ;; Clean pages that have been deleted
-     (when conn
-       (swap! file/*writes (fn [writes]
-                             (->> writes
-                                  (remove (fn [[_ pid]] (d/entity @conn pid)))
-                                  (into {})))))
-     (if (empty? writes)
-       true
-       (do
-         (prn "Unfinished file writes:" @file/*writes)
-         false))))
-
-  (page-file-saved
-   [this request-id page-id]
-   (file/dissoc-request! request-id)
-   nil)
-
-  (sync-app-state
-   [this new-state-str]
-   (let [new-state (ldb/read-transit-str new-state-str)]
-     (worker-state/set-new-state! new-state)
-     nil))
-
-  (sync-ui-state
-   [_this repo state-str]
-   (undo-redo/record-ui-state! repo state-str)
-   nil)
-
-  ;; Export
-  (block->content
-   [this repo block-uuid-str tree->file-opts context]
-   (assert (common-util/uuid-string? block-uuid-str))
-   (let [block-uuid (uuid block-uuid-str)]
-     (when-let [conn (worker-state/get-datascript-conn repo)]
-       (common-file/block->content repo @conn block-uuid
-                                   (ldb/read-transit-str tree->file-opts)
-                                   (ldb/read-transit-str context)))))
-
-  (get-debug-datoms
-   [this repo]
-   (when-let [db (worker-state/get-sqlite-conn repo)]
-     (let [conn (worker-state/get-datascript-conn repo)]
-       (ldb/write-transit-str (worker-export/get-debug-datoms conn db)))))
-
-  (get-all-pages
-   [this repo]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/write-transit-str (worker-export/get-all-pages repo @conn))))
-
-  (get-all-page->content
-   [this repo]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/write-transit-str (worker-export/get-all-page->content repo @conn))))
-
-  ;; RTC
-  (rtc-start
-   [this repo token]
-   (with-write-transit-str
-     (rtc-core/new-task--rtc-start repo token)))
-
-  (rtc-stop
-   [this]
-   (rtc-core/rtc-stop))
-
-  (rtc-toggle-auto-push
-   [this]
-   (rtc-core/rtc-toggle-auto-push))
-
-  (rtc-toggle-remote-profile
-   [this]
-   (rtc-core/rtc-toggle-remote-profile))
-
-  (rtc-grant-graph-access
-   [this token graph-uuid target-user-uuids-str target-user-emails-str]
-   (let [target-user-uuids (ldb/read-transit-str target-user-uuids-str)
-         target-user-emails (ldb/read-transit-str target-user-emails-str)]
-     (with-write-transit-str
-       (rtc-core/new-task--grant-access-to-others token graph-uuid
-                                                  :target-user-uuids target-user-uuids
-                                                  :target-user-emails target-user-emails))))
-
-  (rtc-get-graphs
-   [this token]
-   (with-write-transit-str
-     (rtc-core/new-task--get-graphs token)))
-
-  (rtc-delete-graph
-   [this token graph-uuid schema-version]
-   (with-write-transit-str
-     (rtc-core/new-task--delete-graph token graph-uuid schema-version)))
-
-  (rtc-get-users-info
-   [this token graph-uuid]
-   (with-write-transit-str
-     (rtc-core/new-task--get-users-info token graph-uuid)))
-
-  (rtc-get-block-content-versions
-   [this token graph-uuid block-uuid]
-   (with-write-transit-str
-     (rtc-core/new-task--get-block-content-versions token graph-uuid block-uuid)))
-
-  (rtc-get-debug-state
-   [this]
-   (with-write-transit-str
-     (rtc-core/new-task--get-debug-state)))
-
-  (rtc-async-upload-graph
-   [this repo token remote-graph-name]
-   (with-write-transit-str
-     (rtc-core/new-task--upload-graph token repo remote-graph-name)))
-
-  (rtc-async-branch-graph
-   [this repo token]
-   (with-write-transit-str
-     (rtc-core/new-task--branch-graph token repo)))
-
-  (rtc-request-download-graph
-   [this token graph-uuid schema-version]
-   (with-write-transit-str
-     (rtc-core/new-task--request-download-graph token graph-uuid schema-version)))
-
-  (rtc-wait-download-graph-info-ready
-   [this token download-info-uuid graph-uuid schema-version timeout-ms]
-   (with-write-transit-str
-     (rtc-core/new-task--wait-download-info-ready token download-info-uuid graph-uuid schema-version timeout-ms)))
-
-  (rtc-download-graph-from-s3
-   [this graph-uuid graph-name s3-url]
-   (with-write-transit-str
-     (rtc-core/new-task--download-graph-from-s3 graph-uuid graph-name s3-url)))
-
-  (rtc-download-info-list
-   [this token graph-uuid schema-version]
-   (with-write-transit-str
-     (rtc-core/new-task--download-info-list token graph-uuid schema-version)))
-
-  (rtc-get-graph-keys
-   [this repo]
-   (with-write-transit-str
-     (worker-crypt/get-graph-keys-jwk repo)))
-
-  (rtc-sync-current-graph-encrypted-aes-key
-   [this token device-uuids-transit-str]
-   (with-write-transit-str
-     (worker-device/new-task--sync-current-graph-encrypted-aes-key
-      token device-uuids-transit-str)))
-
-  (device-list-devices
-   [this token]
-   (with-write-transit-str
-     (worker-device/new-task--list-devices token)))
-
-  (device-remove-device-public-key
-   [this token device-uuid key-name]
-   (with-write-transit-str
-     (worker-device/new-task--remove-device-public-key token device-uuid key-name)))
-
-  (device-remove-device
-   [this token device-uuid]
-   (with-write-transit-str
-     (worker-device/new-task--remove-device token device-uuid)))
-
-  (undo
-   [_this repo _page-block-uuid-str]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/write-transit-str (undo-redo/undo repo conn))))
-
-  (redo
-   [_this repo _page-block-uuid-str]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (ldb/write-transit-str (undo-redo/redo repo conn))))
-
-  (record-editor-info
-   [_this repo _page-block-uuid-str editor-info-str]
-   (undo-redo/record-editor-info! repo (ldb/read-transit-str editor-info-str))
-   nil)
-
-  (validate-db
-   [_this repo]
-   (when-let [conn (worker-state/get-datascript-conn repo)]
-     (let [result (worker-db-validate/validate-db @conn)]
-       (db-migrate/fix-db! conn {:invalid-entity-ids (:invalid-entity-ids result)})
-       result)))
-
-  (export-edn
-   [_this repo options]
-   (let [conn (worker-state/get-datascript-conn repo)]
-     (try
-       (->> (ldb/read-transit-str options)
-            (sqlite-export/build-export @conn)
-            ldb/write-transit-str)
-       (catch :default e
-         (js/console.error "export-edn error: " e)
-         (worker-util/post-message :notification
-                                   ["An unexpected error occurred during export. See the javascript console for details."
-                                    :error])))))
-
-  (get-view-data
-   [_this repo view-id opts-str]
-   (let [conn (worker-state/get-datascript-conn repo)
-         data (db-view/get-view-data @conn view-id (ldb/read-transit-str opts-str))]
-     (ldb/write-transit-str data)))
-
-  (get-property-values
-   [_this repo opts-str]
-   (let [conn (worker-state/get-datascript-conn repo)
-         {:keys [property-ident] :as opts} (ldb/read-transit-str opts-str)
-         data (db-view/get-property-values @conn property-ident opts)]
-     (ldb/write-transit-str data)))
-
-  (build-graph
-   [_this repo opts-str]
-   (let [conn (worker-state/get-datascript-conn repo)
-         data (db-graph/build-graph @conn (ldb/read-transit-str opts-str))]
-     (ldb/write-transit-str data)))
-
-  (dangerousRemoveAllDbs
-   [this repo]
-   (p/let [r (.listDB this)
-           dbs (ldb/read-transit-str r)]
-     (p/all (map #(.unsafeUnlinkDB this (:name %)) dbs)))))
+          (worker-util/profile "Worker db transact"
+                               (ldb/transact! conn tx-data' tx-meta')))
+        nil)
+      (catch :default e
+        (prn :debug :error)
+        (js/console.error e)
+        (prn :debug :tx-data @conn tx-data)))))
+
+(def-thread-api :thread-api/get-initial-data
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (sqlite-common-db/get-initial-data @conn)))
+
+(def-thread-api :thread-api/get-page-refs-count
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (sqlite-common-db/get-page->refs-count @conn)))
+
+(def-thread-api :thread-api/close-db
+  [repo]
+  (close-db! repo)
+  nil)
+
+(def-thread-api :thread-api/reset-db
+  [repo db-transit]
+  (reset-db! repo db-transit)
+  nil)
+
+(def-thread-api :thread-api/unsafe-unlink-db
+  [repo]
+  (p/let [pool (<get-opfs-pool repo)
+          _ (close-db! repo)
+          _result (remove-vfs! pool)]
+    nil))
+
+(def-thread-api :thread-api/release-access-handles
+  [repo]
+  (when-let [^js pool (worker-state/get-opfs-pool repo)]
+    (.releaseAccessHandles pool)
+    nil))
+
+(def-thread-api :thread-api/db-exists
+  [repo]
+  (<db-exists? repo))
+
+(def-thread-api :thread-api/export-db
+  [repo]
+  (when-let [^js db (worker-state/get-sqlite-conn repo :db)]
+    (.exec db "PRAGMA wal_checkpoint(2)"))
+  (<export-db-file repo))
+
+(def-thread-api :thread-api/import-db
+  [repo data]
+  (when-not (string/blank? repo)
+    (p/let [pool (<get-opfs-pool repo)]
+      (<import-db pool data)
+      nil)))
+
+(def-thread-api :thread-api/search-blocks
+  [repo q option]
+  (p/let [search-db (get-search-db repo)
+          conn (worker-state/get-datascript-conn repo)]
+    (search/search-blocks repo conn search-db q option)))
+
+(def-thread-api :thread-api/search-upsert-blocks
+  [repo blocks]
+  (p/let [db (get-search-db repo)]
+    (search/upsert-blocks! db (bean/->js blocks))
+    nil))
+
+(def-thread-api :thread-api/search-delete-blocks
+  [repo ids]
+  (p/let [db (get-search-db repo)]
+    (search/delete-blocks! db ids)
+    nil))
+
+(def-thread-api :thread-api/search-truncate-tables
+  [repo]
+  (p/let [db (get-search-db repo)]
+    (search/truncate-table! db)
+    nil))
+
+(def-thread-api :thread-api/search-build-blocks-indice
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (search/build-blocks-indice repo @conn)))
+
+(def-thread-api :thread-api/search-build-pages-indice
+  [_repo]
+  nil)
+
+(def-thread-api :thread-api/apply-outliner-ops
+  [repo ops opts]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (try
+      (worker-util/profile
+       "apply outliner ops"
+       (outliner-op/apply-ops! repo conn ops (worker-state/get-date-formatter repo) opts))
+      (catch :default e
+        (let [data (ex-data e)
+              {:keys [type payload]} (when (map? data) data)]
+          (case type
+            :notification
+            (worker-util/post-message type [(:message payload) (:type payload)])
+            (throw e)))))))
+
+(def-thread-api :thread-api/file-writes-finished?
+  [repo]
+  (let [conn (worker-state/get-datascript-conn repo)
+        writes @file/*writes]
+    ;; Clean pages that have been deleted
+    (when conn
+      (swap! file/*writes (fn [writes]
+                            (->> writes
+                                 (remove (fn [[_ pid]] (d/entity @conn pid)))
+                                 (into {})))))
+    (if (empty? writes)
+      true
+      (do
+        (prn "Unfinished file writes:" @file/*writes)
+        false))))
+
+(def-thread-api :thread-api/page-file-saved
+  [request-id _page-id]
+  (file/dissoc-request! request-id)
+  nil)
+
+(def-thread-api :thread-api/sync-app-state
+  [new-state]
+  (worker-state/set-new-state! new-state)
+  nil)
+
+(def-thread-api :thread-api/sync-ui-state
+  [repo state]
+  (undo-redo/record-ui-state! repo (ldb/write-transit-str state))
+  nil)
+
+(def-thread-api :thread-api/export-get-debug-datoms
+  [repo]
+  (when-let [db (worker-state/get-sqlite-conn repo)]
+    (let [conn (worker-state/get-datascript-conn repo)]
+      (worker-export/get-debug-datoms conn db))))
+
+(def-thread-api :thread-api/export-get-all-pages
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (worker-export/get-all-pages repo @conn)))
+
+(def-thread-api :thread-api/export-get-all-page->content
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (worker-export/get-all-page->content repo @conn)))
+
+(def-thread-api :thread-api/undo
+  [repo _page-block-uuid-str]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (undo-redo/undo repo conn)))
+
+(def-thread-api :thread-api/redo
+  [repo _page-block-uuid-str]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (undo-redo/redo repo conn)))
+
+(def-thread-api :thread-api/record-editor-info
+  [repo _page-block-uuid-str editor-info]
+  (undo-redo/record-editor-info! repo editor-info)
+  nil)
+
+(def-thread-api :thread-api/validate-db
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (let [result (worker-db-validate/validate-db @conn)]
+      (db-migrate/fix-db! conn {:invalid-entity-ids (:invalid-entity-ids result)})
+      result)))
+
+(def-thread-api :thread-api/export-edn
+  [repo options]
+  (let [conn (worker-state/get-datascript-conn repo)]
+    (try
+      (sqlite-export/build-export @conn options)
+      (catch :default e
+        (js/console.error "export-edn error: " e)
+        (worker-util/post-message :notification
+                                  ["An unexpected error occurred during export. See the javascript console for details."
+                                   :error])))))
+
+(def-thread-api :thread-api/get-view-data
+  [repo view-id option]
+  (let [conn (worker-state/get-datascript-conn repo)]
+    (db-view/get-view-data @conn view-id option)))
+
+(def-thread-api :thread-api/get-property-values
+  [repo {:keys [property-ident] :as option}]
+  (let [conn (worker-state/get-datascript-conn repo)]
+    (db-view/get-property-values @conn property-ident option)))
+
+(def-thread-api :thread-api/build-graph
+  [repo option]
+  (let [conn (worker-state/get-datascript-conn repo)]
+    (db-graph/build-graph @conn option)))
+
+(comment
+  (def-thread-api :general/dangerousRemoveAllDbs
+    []
+    (p/let [r (<list-all-dbs)
+            dbs (ldb/read-transit-str r)]
+      (p/all (map #(.unsafeUnlinkDB this (:name %)) dbs)))))
 
 (defn- rename-page!
   [repo conn page-uuid new-name]
@@ -982,13 +788,20 @@
   []
   (glogi-console/install!)
   (check-worker-scope!)
-  (let [^js obj (DBWorker.)]
-    (outliner-register-op-handlers!)
-    (worker-state/set-worker-object! obj)
-    (<ratelimit-file-writes!)
-    (js/setInterval #(.postMessage js/self "keepAliveResponse") (* 1000 25))
-    (Comlink/expose obj)
-    (reset! worker-state/*main-thread (Comlink/wrap js/self))))
+  (outliner-register-op-handlers!)
+  (<ratelimit-file-writes!)
+  (js/setInterval #(.postMessage js/self "keepAliveResponse") (* 1000 25))
+  (Comlink/expose #js{"remoteInvoke" thread-api/remote-function})
+  (let [^js wrapped-main-thread* (Comlink/wrap js/self)
+        wrapped-main-thread (fn [qkw direct-pass-args? & args]
+                              (-> (.remoteInvoke wrapped-main-thread*
+                                                 (str (namespace qkw) "/" (name qkw))
+                                                 direct-pass-args?
+                                                 (if direct-pass-args?
+                                                   (into-array args)
+                                                   (ldb/write-transit-str args)))
+                                  (p/chain ldb/read-transit-str)))]
+    (reset! worker-state/*main-thread wrapped-main-thread)))
 
 (comment
   (defn <remove-all-files!

src/main/frontend/worker/device.cljs

@@ -4,12 +4,13 @@
             [cljs-time.coerce :as tc]
             [cljs-time.core :as t]
             [clojure.string :as string]
+            [frontend.common.missionary :as c.m]
+            [frontend.common.thread-api :refer [def-thread-api]]
             [frontend.worker.crypt :as crypt]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.ws-util :as ws-util]
             [frontend.worker.state :as worker-state]
             [goog.crypt.base64 :as base64]
-            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [missionary.core :as m]
             [promesa.core :as p]))
@@ -172,9 +173,8 @@
         (m/? (new-task--remove-user-device* get-ws-create-task device-uuid*))))))
 
 (defn new-task--sync-current-graph-encrypted-aes-key
-  [token device-uuids-transit-str]
-  (let [repo (worker-state/get-current-repo)
-        device-uuids (ldb/read-transit-str device-uuids-transit-str)]
+  [token device-uuids]
+  (let [repo (worker-state/get-current-repo)]
     (assert (and (seq device-uuids) (every? uuid? device-uuids)) device-uuids)
     (m/sp
       (when-let [graph-uuid (client-op/get-graph-uuid repo)]
@@ -206,3 +206,19 @@
                                  devices*)))]
                 (m/? (new-task--sync-encrypted-aes-key*
                       get-ws-create-task device-uuid->encrypted-aes-key graph-uuid))))))))))
+
+(def-thread-api :thread-api/rtc-sync-current-graph-encrypted-aes-key
+  [token device-uuids]
+  (new-task--sync-current-graph-encrypted-aes-key token device-uuids))
+
+(def-thread-api :thread-api/list-devices
+  [token]
+  (new-task--list-devices token))
+
+(def-thread-api :thread-api/remove-device-public-key
+  [token device-uuid key-name]
+  (new-task--remove-device-public-key token device-uuid key-name))
+
+(def-thread-api :thread-api/remove-device
+  [token device-uuid]
+  (new-task--remove-device token device-uuid))

src/main/frontend/worker/file.cljs

@@ -30,7 +30,7 @@
     (swap! *writes assoc request-id page-id)
     request-id))
 
-(defn- dissoc-request!
+(defn dissoc-request!
   [request-id]
   (when-let [page-id (get @*writes request-id)]
     (let [old-page-request-ids (keep (fn [[r p]]

src/main/frontend/worker/pipeline.cljs

@@ -7,6 +7,7 @@
             [frontend.worker.state :as worker-state]
             [frontend.worker.util :as worker-util]
             [logseq.common.defkeywords :refer [defkeywords]]
+            [logseq.common.uuid :as common-uuid]
             [logseq.db :as ldb]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.export :as sqlite-export]
@@ -49,6 +50,8 @@
 (defn- insert-tag-templates
   [repo conn tx-report]
   (let [db (:db-after tx-report)
+        journal-id (:db/id (d/entity db :logseq.class/Journal))
+        journal-template? (some (fn [d] (and (:added d) (= (:a d) :block/tags) (= (:v d) journal-id))) (:tx-data tx-report))
         tx-data (some->> (:tx-data tx-report)
                          (filter (fn [d] (and (= (:a d) :block/tags) (:added d))))
                          (group-by :e)
@@ -56,9 +59,12 @@
                                    (let [object (d/entity db e)
                                          template-blocks (->> (mapcat (fn [id]
                                                                         (let [tag (d/entity db id)
+                                                                              journal? (= journal-id id)
                                                                               parents (ldb/get-page-parents tag {:node-class? true})
                                                                               templates (mapcat :logseq.property/_template-applied-to (conj parents tag))]
-                                                                          templates))
+                                                                          (cond->> templates
+                                                                            journal?
+                                                                            (map (fn [t] (assoc t :journal tag))))))
                                                                       (set (map :v datoms)))
                                                               distinct
                                                               (sort-by :block/created-at)
@@ -68,10 +74,17 @@
                                                                               blocks (->>
                                                                                       (cons (assoc (first template-blocks) :logseq.property/used-template (:db/id template))
                                                                                             (rest template-blocks))
-                                                                                      (map (fn [e] (assoc (into {} e) :db/id (:db/id e)))))]
+                                                                                      (map (fn [e]
+                                                                                             (cond->
+                                                                                              (assoc (into {} e) :db/id (:db/id e))
+                                                                                               (:journal template)
+                                                                                               (assoc :block/uuid
+                                                                                                      (common-uuid/gen-journal-template-block (:block/uuid (:journal template))
+                                                                                                                                              (:block/uuid e)))))))]
                                                                           blocks))))]
                                      (when (seq template-blocks)
-                                       (let [result (outliner-core/insert-blocks repo conn template-blocks object {:sibling? false})]
+                                       (let [result (outliner-core/insert-blocks repo conn template-blocks object {:sibling? false
+                                                                                                                   :keep-uuid? journal-template?})]
                                          (:tx-data result)))))))]
     tx-data))
 

src/main/frontend/worker/rtc/asset.cljs

@@ -13,7 +13,6 @@
             [frontend.worker.rtc.ws-util :as ws-util]
             [frontend.worker.state :as worker-state]
             [logseq.common.path :as path]
-            [logseq.db :as ldb]
             [malli.core :as ma]
             [missionary.core :as m])
   (:import [missionary Cancelled]))
@@ -46,9 +45,9 @@
   "Return nil if this asset not exist"
   [repo block-uuid asset-type]
   (m/sp
-    (ldb/read-transit-str
-     (c.m/<?
-      (.get-asset-file-metadata ^js @worker-state/*main-thread repo (str block-uuid) asset-type)))))
+    (c.m/<?
+     (worker-state/<invoke-main-thread :thread-api/get-asset-file-metadata
+                                       repo (str block-uuid) asset-type))))
 
 (defn- remote-block-ops=>remote-asset-ops
   [db-before remove-ops]
@@ -133,11 +132,10 @@
   [repo asset-uuid->url asset-uuid->asset-type]
   (->> (fn [[asset-uuid url]]
          (m/sp
-           (let [r (ldb/read-transit-str
-                    (c.m/<?
-                     (.rtc-download-asset
-                      ^js @worker-state/*main-thread
-                      repo (str asset-uuid) (get asset-uuid->asset-type asset-uuid) url)))]
+           (let [r (c.m/<?
+                    (worker-state/<invoke-main-thread :thread-api/rtc-download-asset
+                                                      repo (str asset-uuid)
+                                                      (get asset-uuid->asset-type asset-uuid) url))]
              (when-let [edata (:ex-data r)]
                ;; if download-url return 404, ignore this asset
                (when (not= 404 (:status (:data edata)))
@@ -151,11 +149,9 @@
   (->> (fn [[asset-uuid url]]
          (m/sp
            (let [[asset-type checksum] (get asset-uuid->asset-type+checksum asset-uuid)
-                 r (ldb/read-transit-str
-                    (c.m/<?
-                     (.rtc-upload-asset
-                      ^js @worker-state/*main-thread
-                      repo (str asset-uuid) asset-type checksum url)))]
+                 r (c.m/<?
+                    (worker-state/<invoke-main-thread :thread-api/rtc-upload-asset
+                                                      repo (str asset-uuid) asset-type checksum url))]
              (when (:ex-data r)
                (throw (ex-info "upload asset failed" r)))
              (d/transact! conn
@@ -244,7 +240,8 @@
                                             :asset-uuids (keys asset-uuid->asset-type)}))
                    :asset-uuid->url))]
         (doseq [[asset-uuid asset-type] remove-asset-uuid->asset-type]
-          (c.m/<? (.unlinkAsset ^js @worker-state/*main-thread repo (str asset-uuid) asset-type)))
+          (c.m/<? (worker-state/<invoke-main-thread :thread-api/unlink-asset
+                                                    repo (str asset-uuid) asset-type)))
         (when (seq asset-uuid->url)
           (add-log-fn :rtc.asset.log/download-assets {:asset-uuids (keys asset-uuid->url)}))
         (m/? (new-task--concurrent-download-assets repo asset-uuid->url asset-uuid->asset-type))))))
@@ -263,8 +260,8 @@
 (defn- new-task--initial-download-missing-assets
   [repo get-ws-create-task graph-uuid conn add-log-fn]
   (m/sp
-    (let [local-all-asset-file-paths (ldb/read-transit-str
-                                      (c.m/<? (.get-all-asset-file-paths ^js @worker-state/*main-thread repo)))
+    (let [local-all-asset-file-paths
+          (c.m/<? (worker-state/<invoke-main-thread :thread-api/get-all-asset-file-paths repo))
           local-all-asset-file-uuids (set (map (comp parse-uuid path/file-stem) local-all-asset-file-paths))
           local-all-asset-uuids (set (map :block/uuid (get-all-asset-blocks @conn)))]
       (when-let [asset-update-ops

src/main/frontend/worker/rtc/client_op.cljs

@@ -168,7 +168,8 @@
                (-> r
                    (update block-uuid assoc :remove :retract)
                    (update-in [block-uuid :update] (fn [old-op]
-                                                     (if old-op
+                                                     (if (and old-op
+                                                              (not (keyword-identical? :retract old-op)))
                                                        (merge-update-ops old-op op)
                                                        op))))
                :remove

src/main/frontend/worker/rtc/core.cljs

@@ -3,6 +3,7 @@
   (:require [clojure.data :as data]
             [datascript.core :as d]
             [frontend.common.missionary :as c.m]
+            [frontend.common.thread-api :refer [def-thread-api]]
             [frontend.worker.device :as worker-device]
             [frontend.worker.rtc.asset :as r.asset]
             [frontend.worker.rtc.branch-graph :as r.branch-graph]
@@ -530,6 +531,77 @@
 
 (def new-task--download-graph-from-s3 r.upload-download/new-task--download-graph-from-s3)
 
+(def-thread-api :thread-api/rtc-start
+  [repo token]
+  (new-task--rtc-start repo token))
+
+(def-thread-api :thread-api/rtc-stop
+  []
+  (rtc-stop))
+
+(def-thread-api :thread-api/rtc-toggle-auto-push
+  []
+  (rtc-toggle-auto-push))
+
+(def-thread-api :thread-api/rtc-toggle-remote-profile
+  []
+  (rtc-toggle-remote-profile))
+
+(def-thread-api :thread-api/rtc-grant-graph-access
+  [token graph-uuid target-user-uuids target-user-emails]
+  (new-task--grant-access-to-others token graph-uuid
+                                    :target-user-uuids target-user-uuids
+                                    :target-user-emails target-user-emails))
+
+(def-thread-api :thread-api/rtc-get-graphs
+  [token]
+  (new-task--get-graphs token))
+
+(def-thread-api :thread-api/rtc-delete-graph
+  [token graph-uuid schema-version]
+  (new-task--delete-graph token graph-uuid schema-version))
+
+(def-thread-api :thread-api/rtc-get-users-info
+  [token graph-uuid]
+  (new-task--get-users-info token graph-uuid))
+
+(def-thread-api :thread-api/rtc-get-block-content-versions
+  [token graph-uuid block-uuid]
+  (new-task--get-block-content-versions token graph-uuid block-uuid))
+
+(def-thread-api :thread-api/rtc-get-debug-state
+  []
+  (new-task--get-debug-state))
+
+(def-thread-api :thread-api/rtc-async-upload-graph
+  [repo token remote-graph-name]
+  (new-task--upload-graph token repo remote-graph-name))
+
+(def-thread-api :thread-api/rtc-async-branch-graph
+  [repo token]
+  (new-task--branch-graph token repo))
+
+(def-thread-api :thread-api/rtc-request-download-graph
+  [token graph-uuid schema-version]
+  (new-task--request-download-graph token graph-uuid schema-version))
+
+(def-thread-api :thread-api/rtc-wait-download-graph-info-ready
+  [token download-info-uuid graph-uuid schema-version timeout-ms]
+  (new-task--wait-download-info-ready token download-info-uuid graph-uuid schema-version timeout-ms))
+
+(def-thread-api :thread-api/rtc-download-graph-from-s3
+  [graph-uuid graph-name s3-url]
+  (new-task--download-graph-from-s3 graph-uuid graph-name s3-url))
+
+(def-thread-api :thread-api/rtc-download-info-list
+  [token graph-uuid schema-version]
+  (new-task--download-info-list token graph-uuid schema-version))
+
+(def-thread-api :thread-api/rtc-add-migration-client-ops
+  [repo server-schema-version]
+  (when-let [db @(worker-state/get-datascript-conn repo)]
+    (add-migration-client-ops! repo db server-schema-version)))
+
 ;;; ================ API (ends) ================
 
 ;;; subscribe state ;;;

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -5,8 +5,10 @@
             [clojure.set :as set]
             [datascript.core :as d]
             [frontend.common.missionary :as c.m]
+            [frontend.common.thread-api :as thread-api]
             [frontend.worker.crypt :as crypt]
             [frontend.worker.db-listener :as db-listener]
+            [frontend.worker.db-metadata :as worker-db-metadata]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
@@ -164,8 +166,7 @@
             (client-op/add-all-exists-asset-as-ops repo)
             (crypt/store-graph-keys-jwk repo aes-key-jwk)
             (when-not rtc-const/RTC-E2E-TEST
-              (let [^js worker-obj (:worker/object @worker-state/*state)]
-                (c.m/<? (.storeMetadata worker-obj repo (pr-str {:kv/value graph-uuid})))))
+              (c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid}))))
             (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-completed
                                                         :message "upload-graph completed"})
             {:graph-uuid graph-uuid})
@@ -347,8 +348,7 @@
         tx-data (concat
                  (blocks-resolve-temp-id normal-blocks)
                  [(ldb/kv :logseq.kv/graph-uuid graph-uuid)])
-        init-tx-data (cons (ldb/kv :logseq.kv/db-type "db") schema-blocks)
-        ^js worker-obj (:worker/object @worker-state/*state)]
+        init-tx-data (cons (ldb/kv :logseq.kv/db-type "db") schema-blocks)]
     (m/sp
       (client-op/update-local-tx repo t)
       (rtc-log-and-state/update-local-t graph-uuid t)
@@ -357,16 +357,20 @@
         (create-graph-for-rtc-test repo init-tx-data tx-data)
         (c.m/<?
          (p/do!
-          (.createOrOpenDB worker-obj repo (ldb/write-transit-str {:close-other-db? false}))
-          (.exportDB worker-obj repo)
-          (.transact worker-obj repo init-tx-data {:rtc-download-graph? true
-                                                   :gen-undo-ops? false
-                                                    ;; only transact db schema, skip validation to avoid warning
-                                                   :frontend.worker.pipeline/skip-validate-db? true
-                                                   :persist-op? false} (worker-state/get-context))
-          (.transact worker-obj repo tx-data {:rtc-download-graph? true
-                                              :gen-undo-ops? false
-                                              :persist-op? false} (worker-state/get-context))
+          ((@thread-api/*thread-apis :thread-api/create-or-open-db) repo {:close-other-db? false})
+          ((@thread-api/*thread-apis :thread-api/export-db) repo)
+          ((@thread-api/*thread-apis :thread-api/transact)
+           repo init-tx-data
+           {:rtc-download-graph? true
+            :gen-undo-ops? false
+            ;; only transact db schema, skip validation to avoid warning
+            :frontend.worker.pipeline/skip-validate-db? true
+            :persist-op? false}
+           (worker-state/get-context))
+          ((@thread-api/*thread-apis :thread-api/transact)
+           repo tx-data {:rtc-download-graph? true
+                         :gen-undo-ops? false
+                         :persist-op? false} (worker-state/get-context))
           (transact-remote-schema-version! repo)
           (transact-block-refs! repo))))
       (worker-util/post-message :add-repo {:repo repo}))))
@@ -423,8 +427,7 @@
     (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :downloading-graph-data
                                                   :message "downloading graph data"
                                                   :graph-uuid graph-uuid})
-    (let [^js worker-obj              (:worker/object @worker-state/*state)
-          {:keys [status body] :as r} (m/? (http/get s3-url {:with-credentials? false}))
+    (let [{:keys [status body] :as r} (m/? (http/get s3-url {:with-credentials? false}))
           repo                        (str sqlite-util/db-version-prefix graph-name)]
       (if (not= 200 status)
         (throw (ex-info "download-graph from s3 failed" {:resp r}))
@@ -437,7 +440,7 @@
             (m/? (new-task--transact-remote-all-blocks all-blocks repo graph-uuid))
             (client-op/update-graph-uuid repo graph-uuid)
             (when-not rtc-const/RTC-E2E-TEST
-              (c.m/<? (.storeMetadata worker-obj repo (pr-str {:kv/value graph-uuid}))))
+              (c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid}))))
             (worker-state/set-rtc-downloading-graph! false)
             (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :download-completed
                                                           :message "download completed"
@@ -482,8 +485,7 @@
             (client-op/remove-local-tx repo)
             (client-op/add-all-exists-asset-as-ops repo)
             (crypt/store-graph-keys-jwk repo aes-key-jwk)
-            (let [^js worker-obj (:worker/object @worker-state/*state)]
-              (c.m/<? (.storeMetadata worker-obj repo (pr-str {:kv/value graph-uuid}))))
+            (c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid})))
             (rtc-log-and-state/rtc-log :rtc.log/branch-graph {:sub-type :completed
                                                               :message "branch-graph completed"})
             nil)

src/main/frontend/worker/rtc/remote_update.cljs

@@ -440,11 +440,14 @@ so need to pull earlier remote-data from websocket."})
     (concat tx-data1 tx-data2)))
 
 (defn- remote-op-value->tx-data
-  [db ent op-value]
+  "ignore-attr-set: don't update local attrs in this set"
+  [db ent op-value ignore-attr-set]
   (assert (some? (:db/id ent)) ent)
   (let [db-schema (d/schema db)
         local-block-map (->> ent
-                             (filter (comp update-op-watched-attr? first))
+                             (filter (fn [[attr _]]
+                                       (and (update-op-watched-attr? attr)
+                                            (not (contains? ignore-attr-set attr)))))
                              (keep (fn [[k v]]
                                      (when-let [[ref? card-many?] (get-schema-ref+cardinality db-schema k)]
                                        [k
@@ -496,7 +499,8 @@ so need to pull earlier remote-data from websocket."})
           (upsert-whiteboard-block repo conn op-value)
           (do (when-let [schema-tx-data (remote-op-value->schema-tx-data block-uuid op-value)]
                 (ldb/transact! conn schema-tx-data {:persist-op? false :gen-undo-ops? false}))
-              (when-let [tx-data (seq (remote-op-value->tx-data @conn ent (dissoc op-value :client/schema)))]
+              (when-let [tx-data (seq (remote-op-value->tx-data @conn ent (dissoc op-value :client/schema)
+                                                                rtc-const/ignore-attrs-when-syncing))]
                 (ldb/transact! conn (concat tx-data update-block-order-tx-data)
                                {:persist-op? false :gen-undo-ops? false}))))))))
 

src/main/frontend/worker/search.cljs

@@ -332,7 +332,7 @@ DROP TRIGGER IF EXISTS blocks_au;
   (drop-tables-and-triggers! db)
   (create-tables-and-triggers! db))
 
-(defn get-all-block-contents
+(defn get-all-blocks
   [db]
   (when db
     (->> (d/datoms db :avet :block/uuid)
@@ -345,9 +345,8 @@ DROP TRIGGER IF EXISTS blocks_au;
 (defn build-blocks-indice
   [repo db]
   (build-fuzzy-search-indice repo db)
-  (->> (get-all-block-contents db)
-       (keep block->index)
-       (bean/->js)))
+  (->> (get-all-blocks db)
+       (keep block->index)))
 
 (defn- get-blocks-from-datoms-impl
   [repo {:keys [db-after db-before]} datoms]

src/main/frontend/worker/state.cljs

@@ -10,9 +10,27 @@
 
 (defonce *main-thread (atom nil))
 
-(defonce *state (atom {:worker/object nil
-
-                       :db/latest-transact-time {}
+(defn- <invoke-main-thread*
+  [qkw direct-pass-args? args-list]
+  (let [main-thread @*main-thread]
+    (when (nil? main-thread)
+      (prn :<invoke-main-thread-error qkw)
+      (throw (ex-info "main-thread has not been initialized" {})))
+    (apply main-thread qkw direct-pass-args? args-list)))
+
+(defn <invoke-main-thread
+  "invoke main thread api"
+  [qkw & args]
+  (<invoke-main-thread* qkw false args))
+
+(comment
+  (defn <invoke-main-thread-direct-pass-args
+    "invoke main thread api.
+  But directly pass args to main-thread(won't do transit-write on them)"
+    [qkw & args]
+    (<invoke-main-thread* qkw true args)))
+
+(defonce *state (atom {:db/latest-transact-time {}
                        :worker/context {}
 
                        ;; FIXME: this name :config is too general
@@ -102,14 +120,6 @@
   (swap! *state (fn [old-state]
                   (merge old-state new-state))))
 
-(defn set-worker-object!
-  [worker]
-  (swap! *state assoc :worker/object worker))
-
-(defn get-worker-object
-  []
-  (:worker/object @*state))
-
 (defn get-date-formatter
   [repo]
   (common-config/get-date-formatter (get-config repo)))

src/main/logseq/api.cljs

@@ -793,8 +793,7 @@
             (some-> (editor-handler/insert-block-tree-after-target
                      (:db/id block) sibling bb (get block :block/format :markdown) keep-uuid?)
                     (p/then (fn [results]
-                              (some-> results (ldb/read-transit-str)
-                                      :blocks (sdk-utils/normalize-keyword-for-json) (bean/->js)))))))))))
+                              (some-> results :blocks (sdk-utils/normalize-keyword-for-json) (bean/->js)))))))))))
 
 (def ^:export remove_block
   (fn [block-uuid ^js _opts]
@@ -903,7 +902,10 @@
   [k]
   (this-as this
            (p/let [prop (-get-property this k)]
-             (bean/->js (sdk-utils/normalize-keyword-for-json prop)))))
+             (-> prop
+                 (assoc :type (:logseq.property/type prop))
+                 (sdk-utils/normalize-keyword-for-json)
+                 (bean/->js)))))
 
 (defn ^:export upsert_property
   "schema:
@@ -929,7 +931,8 @@
                                 (string? (:cardinality schema))
                                 (update :cardinality keyword)
                                 (string? (:type schema))
-                                (update :type keyword))
+                                (-> (assoc :logseq.property/type (keyword (:type schema)))
+                                    (dissoc :type)))
                        p (db-property-handler/upsert-property! k schema
                                                                (cond-> opts
                                                                  name

src/resources/dicts/en.edn

@@ -462,6 +462,7 @@
  :export-opml "Export as OPML"
  :export-public-pages "Export public pages"
  :export-json "Export as JSON"
+ :export-db-edn "Export EDN file"
  :export-sqlite-db "Export SQLite DB"
  :export-zip "Export both SQLite DB and assets"
  :export-roam-json "Export as Roam JSON"

src/test/frontend/worker/rtc/remote_update_test.cljs

@@ -15,7 +15,7 @@
                                :block/title "local-content"}])
             op-value {:block/title (ldb/write-transit-str "remote-content")}]
         (is (= [[:db/add (:db/id (d/entity db [:block/uuid block-uuid])) :block/title "remote-content"]]
-               (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value)))))
+               (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value nil)))))
 
     (testing ":block/tags (1)"
       (let [db (d/db-with db [{:block/uuid block-uuid}
@@ -26,7 +26,7 @@
                                                                     [:block/uuid ref-uuid1]
                                                                     [:block/uuid ref-uuid2]]))]
         (is (= #{[:db/add db-id :block/tags ref1] [:db/add db-id :block/tags ref2]}
-               (set (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value))))))
+               (set (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value nil))))))
 
     (testing ":block/tags (2)"
       (let [db (d/db-with db [{:db/id "ref1"
@@ -39,7 +39,7 @@
                                                                [:block/uuid ref-uuid2]]))]
         (is (= #{[:db/retract db-id :block/tags [:block/uuid ref-uuid1]]
                  [:db/add db-id :block/tags ref2]}
-               (set (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value))))))
+               (set (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value nil))))))
 
     (testing ":block/tags (3): ref2 not exist"
       (let [db (d/db-with db [{:db/id "ref1"
@@ -48,13 +48,13 @@
                                :block/tags ["ref1"]}])
             op-value {:block/tags [ref-uuid2]}]
         (is (= #{[:db/retract (:db/id (d/entity db [:block/uuid block-uuid])) :block/tags [:block/uuid ref-uuid1]]}
-               (set (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value))))))
+               (set (#'subject/remote-op-value->tx-data db (d/entity db [:block/uuid block-uuid]) op-value nil))))))
     (testing ":block/updated-at"
       (let [db (d/db-with db [{:block/uuid block-uuid
                                :block/updated-at 1}])
             ent (d/entity db [:block/uuid block-uuid])]
         (is (= [[:db/retract (:db/id ent) :block/updated-at]]
-               (#'subject/remote-op-value->tx-data db ent {})))))
+               (#'subject/remote-op-value->tx-data db ent {} nil)))))
     (testing ":logseq.task/status, op-value don't have this attr, means remove this attr"
       (let [db (d/db-with db [{:db/id "ref1"
                                :block/uuid ref-uuid1}
@@ -63,4 +63,13 @@
             op-value {}
             ent (d/entity db [:block/uuid block-uuid])]
         (is (= [[:db/retract (:db/id ent) :logseq.task/status]]
-               (#'subject/remote-op-value->tx-data db ent op-value)))))))
+               (#'subject/remote-op-value->tx-data db ent op-value nil)))))
+    (testing "dont update ignored attrs"
+      (let [db (d/db-with db [{:block/uuid block-uuid
+                               :logseq.property.view/feature-type :aaa}])
+            op-value {}
+            ent (d/entity db [:block/uuid block-uuid])
+            ignore-attr-set #{:logseq.property.view/feature-type}]
+        (is (empty? (#'subject/remote-op-value->tx-data db ent op-value ignore-attr-set)))
+        (is (= [[:db/retract (:db/id ent) :logseq.property.view/feature-type]]
+               (#'subject/remote-op-value->tx-data db ent op-value nil)))))))