Merge pull request #11784 from logseq/feat/export-graph-edn

feat: export and import graph as edn

.clj-kondo/config.edn

@@ -102,6 +102,7 @@
              frontend.handler.common.plugin plugin-common-handler
              frontend.handler.config config-handler
              frontend.handler.db-based.editor db-editor-handler
+             frontend.handler.db-based.export db-export-handler
              frontend.handler.db-based.page db-page-handler
              frontend.handler.db-based.property db-property-handler
              frontend.handler.db-based.property.util db-pu

.github/workflows/build.yml

@@ -165,17 +165,23 @@ jobs:
         run: cd scripts && yarn install --frozen-lockfile
 
       - name: Create DB graph with properties
-        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs ./db-graph-with-props
+        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_properties.cljs ./properties-graph
 
       # TODO: Use a smaller, test-focused graph to test classes
       - name: Create DB graph with classes
-        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs ./db-graph-with-schema
+        run: cd scripts && yarn nbb-logseq src/logseq/tasks/db_graph/create_graph_with_schema_org.cljs ./schema-graph
 
       - name: Fetch deps/db yarn deps
         run: cd deps/db && yarn install --frozen-lockfile
 
       - name: Validate created DB graphs
-        run: cd deps/db && yarn nbb-logseq script/validate_db.cljs ../../scripts/db-graph-with-props ../../scripts/db-graph-with-schema --closed-maps --group-errors
+        run: cd deps/db && yarn nbb-logseq script/validate_db.cljs ../../scripts/properties-graph ../../scripts/schema-graph --closed-maps --group-errors
+
+      - name: Export a created DB graph
+        run: cd deps/db && yarn nbb-logseq script/export_graph.cljs ../../scripts/properties-graph -f properties.edn -t
+
+      - name: Create graph from the export and diff the two graphs
+        run: cd deps/db && yarn nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs ./properties-graph2 properties.edn -iv && yarn nbb-logseq script/diff_graphs.cljs ../../scripts/properties-graph ./properties-graph2 -t
 
   e2e-test:
     # TODO: Re-enable when ready to enable tests for file graphs

bb.edn

@@ -85,6 +85,18 @@
    :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
                 "yarn -s nbb-logseq -cp src:../outliner/src:script script/create_graph.cljs" *command-line-args*)}
 
+  dev:db-export
+  {:doc "Export a DB graph to a sqlite.build EDN file"
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq script/export_graph.cljs" *command-line-args*)}
+
+  dev:db-diff
+  {:doc "Diffs two DB graphs"
+   :requires ([babashka.fs :as fs])
+   :task (apply shell {:dir "deps/db" :extra-env {"ORIGINAL_PWD" (fs/cwd)}}
+                "yarn -s nbb-logseq script/diff_graphs.cljs" *command-line-args*)}
+
   dev:db-import
   {:doc "Import a file graph to db graph"
    :requires ([babashka.fs :as fs])

deps/db/script/create_graph.cljs

@@ -1,13 +1,15 @@
 (ns create-graph
-  "A script that creates a DB graph given a sqlite.build EDN file"
+  "A script that creates or updates a DB graph given a sqlite.build EDN file.
+   If the given graph already exists, the EDN file updates the graph."
   (:require ["fs" :as fs]
             ["os" :as os]
             ["path" :as node-path]
-            #_:clj-kondo/ignore
             [babashka.cli :as cli]
             [clojure.edn :as edn]
             [clojure.string :as string]
             [datascript.core :as d]
+            [logseq.db.sqlite.export :as sqlite-export]
+            #_:clj-kondo/ignore
             [logseq.outliner.cli :as outliner-cli]
             [nbb.classpath :as cp]
             [nbb.core :as nbb]
@@ -36,7 +38,9 @@
   {:help {:alias :h
           :desc "Print help"}
    :validate {:alias :v
-              :desc "Validate db after creation"}})
+              :desc "Validate db after creation"}
+   :import {:alias :i
+            :desc "Import edn file using sqlite-export"}})
 
 (defn -main [args]
   (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
@@ -46,16 +50,21 @@
                           (cli/format-opts {:spec spec})))
             (js/process.exit 1))
         [dir db-name] (get-dir-and-db-name graph-dir)
-        sqlite-build-edn (merge {:auto-create-ontology? true}
+        sqlite-build-edn (merge (if (:import options) {} {:auto-create-ontology? true})
                                 (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
+        graph-exists? (fs/existsSync (node-path/join dir db-name))
         conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph})
-        {:keys [init-tx block-props-tx] :as _txs} (outliner-cli/build-blocks-tx sqlite-build-edn)]
+        {:keys [init-tx block-props-tx misc-tx] :as _txs}
+        (if (:import options)
+          (sqlite-export/build-import sqlite-build-edn @conn {})
+          (outliner-cli/build-blocks-tx sqlite-build-edn))]
     (println "Generating" (count (filter :block/name init-tx)) "pages and"
              (count (filter :block/title init-tx)) "blocks ...")
     ;; (cljs.pprint/pprint _txs)
     (d/transact! conn init-tx)
-    (d/transact! conn block-props-tx)
-    (println "Created graph" (str db-name "!"))
+    (when (seq block-props-tx) (d/transact! conn block-props-tx))
+    (when (seq misc-tx) (d/transact! conn misc-tx))
+    (println (if graph-exists? "Updated graph" "Created graph") (str db-name "!"))
     (when (:validate options)
       (validate-db/validate-db @conn db-name {:group-errors true :closed-maps true :humanize true}))))
 

deps/db/script/diff_graphs.cljs

@@ -0,0 +1,74 @@
+(ns diff-graphs
+  "A script that diffs two DB graphs through their sqlite.build EDN"
+  (:require ["os" :as os]
+            ["path" :as node-path]
+            [babashka.cli :as cli]
+            [clojure.data :as data]
+            [clojure.pprint :as pprint]
+            [clojure.string :as string]
+            [logseq.common.config :as common-config]
+            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [nbb.core :as nbb]))
+
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
+(def spec
+  "Options spec"
+  {:help {:alias :h
+          :desc "Print help"}
+   :exclude-namespaces {:alias :e
+                        :coerce #{}
+                        :desc "Namespaces to exclude from properties and classes"}
+   :exclude-built-in-pages? {:alias :b
+                             :desc "Exclude built-in pages"}
+   :set-diff {:alias :s
+              :desc "Use set to reduce noisy diff caused by ordering"}
+   :include-timestamps? {:alias :t
+                         :desc "Include timestamps in export"}})
+
+(defn -main [args]
+  (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
+        [graph-dir graph-dir2] args'
+        _ (when (or (nil? graph-dir) (nil? graph-dir2) (:help options))
+            (println (str "Usage: $0 GRAPH-NAME GRAPH-NAME2 [& ARGS] [OPTIONS]\nOptions:\n"
+                          (cli/format-opts {:spec spec})))
+            (js/process.exit 1))
+        conn (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir))
+        conn2 (apply sqlite-cli/open-db! (get-dir-and-db-name graph-dir2))
+        export-options (select-keys options [:include-timestamps? :exclude-namespaces :exclude-built-in-pages?])
+        export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})
+        export-map2 (sqlite-export/build-export @conn2 {:export-type :graph :graph-options export-options})
+        prepare-export-to-diff
+        (fn [m]
+          (cond->
+           (-> m
+               (update :classes update-vals (fn [m]
+                                              (update m :build/class-properties sort)))
+               (update ::sqlite-export/kv-values
+                       (fn [kvs]
+                         ;; Ignore extra metadata that a copied graph can add
+                         (vec (remove #(#{:logseq.kv/import-type :logseq.kv/imported-at} (:db/ident %)) kvs))))
+              ;; TODO: fix built-in views for schema export
+               (update :pages-and-blocks (fn [pbs]
+                                           (vec (remove #(= (:block/title (:page %)) common-config/views-page-name) pbs)))))
+            (:set-diff options)
+            (update-vals set)))
+        diff (->> (data/diff (prepare-export-to-diff export-map) (prepare-export-to-diff export-map2))
+                  butlast)]
+    (if (= diff [nil nil])
+      (println "The two graphs are equal!")
+      (do (pprint/pprint diff)
+          (js/process.exit 1)))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/script/export_graph.cljs

@@ -0,0 +1,70 @@
+(ns export-graph
+  "A script that exports a graph to a sqlite.build EDN file"
+  (:require ["fs" :as fs]
+            ["os" :as os]
+            ["path" :as node-path]
+            [babashka.cli :as cli]
+            [clojure.edn :as edn]
+            [clojure.pprint :as pprint]
+            [clojure.string :as string]
+            [logseq.db.sqlite.cli :as sqlite-cli]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [nbb.core :as nbb]))
+
+(defn- resolve-path
+  "If relative path, resolve with $ORIGINAL_PWD"
+  [path]
+  (if (node-path/isAbsolute path)
+    path
+    (node-path/join (or js/process.env.ORIGINAL_PWD ".") path)))
+
+(defn- get-dir-and-db-name
+  "Gets dir and db name for use with open-db! Works for relative and absolute paths and
+   defaults to ~/logseq/graphs/ when no '/' present in name"
+  [graph-dir]
+  (if (string/includes? graph-dir "/")
+    (let [resolve-path' #(if (node-path/isAbsolute %) %
+                             ;; $ORIGINAL_PWD used by bb tasks to correct current dir
+                             (node-path/join (or js/process.env.ORIGINAL_PWD ".") %))]
+      ((juxt node-path/dirname node-path/basename) (resolve-path' graph-dir)))
+    [(node-path/join (os/homedir) "logseq" "graphs") graph-dir]))
+
+(def spec
+  "Options spec"
+  {:help {:alias :h
+          :desc "Print help"}
+   :include-timestamps? {:alias :t
+                         :desc "Include timestamps in export"}
+   :file {:alias :f
+          :desc "Saves edn to file"}
+   :exclude-namespaces {:alias :e
+                        :coerce #{}
+                        :desc "Namespaces to exclude from properties and classes"}
+   :exclude-built-in-pages? {:alias :b
+                             :desc "Exclude built-in pages"}
+   :export-options {:alias :E
+                    :desc "Raw options map to pass to export"}})
+
+(defn -main [args]
+  (let [{options :opts args' :args} (cli/parse-args args {:spec spec})
+        graph-dir (first args')
+        _ (when (or (nil? graph-dir) (:help options))
+            (println (str "Usage: $0 GRAPH-NAME [& ARGS] [OPTIONS]\nOptions:\n"
+                          (cli/format-opts {:spec spec})))
+            (js/process.exit 1))
+        [dir db-name] (get-dir-and-db-name graph-dir)
+        conn (sqlite-cli/open-db! dir db-name)
+        export-options (merge (select-keys options [:include-timestamps? :exclude-namespaces :exclude-built-in-pages?])
+                              (edn/read-string (:export-options options)))
+        export-map (sqlite-export/build-export @conn {:export-type :graph :graph-options export-options})]
+    (if (:file options)
+      (do
+        (println "Exported" (count (:properties export-map)) "properties,"
+                 (count (:properties export-map)) "classes and"
+                 (count (:pages-and-blocks export-map)) "pages")
+        (fs/writeFileSync (resolve-path (:file options))
+                          (with-out-str (pprint/pprint export-map))))
+      (pprint/pprint export-map))))
+
+(when (= nbb/*file* (nbb/invoked-file))
+  (-main *command-line-args*))

deps/db/src/logseq/db/frontend/db_ident.cljc

@@ -68,7 +68,7 @@
          :cljs (exists? js/process)
          :default false)
     ;; So that we don't have to change :user.{property|class} in our tests
-    (keyword user-namespace (-> name-string (string/replace #"/|\s+" "-") (string/replace-first #"^(\d)" "NUM-$1")))
+    (keyword user-namespace (-> name-string (string/replace #"[/()]|\s+" "-") (string/replace-first #"^(\d)" "NUM-$1")))
     (keyword user-namespace
              (str
               (->> (filter #(re-find #"[0-9a-zA-Z-]{1}" %) (seq name-string)) (apply str))

deps/db/src/logseq/db/sqlite/build.cljs

@@ -8,6 +8,7 @@
   (:require [cljs.pprint :as pprint]
             [clojure.set :as set]
             [clojure.string :as string]
+            [clojure.walk :as walk]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
@@ -46,6 +47,8 @@
           [:block/uuid page-uuid]
           (throw (ex-info (str "No uuid for page '" (second val) "'") {:name (second val)}))))
       :block/uuid
+      val
+      ;; Allow through :coll properties like
       val)
     val))
 
@@ -68,17 +71,19 @@
     (or (get all-idents kw)
         (throw (ex-info (str "No ident found for " (pr-str kw)) {})))))
 
-(defn- ->block-properties [properties page-uuids all-idents]
-  (->>
-   (map
-    (fn [[prop-name val]]
-      [(get-ident all-idents prop-name)
-       ;; set indicates a :many value
-       (if (set? val)
-         (set (map #(translate-property-value % page-uuids) val))
-         (translate-property-value val page-uuids))])
-    properties)
-   (into {})))
+(defn- ->block-properties [properties page-uuids all-idents {:keys [translate-property-values?]}]
+  (let [translate-property-values (if translate-property-values?
+                                    (fn translate-property-values [val]
+                                      ;; set indicates a :many value
+                                      (if (set? val)
+                                        (set (map #(translate-property-value % page-uuids) val))
+                                        (translate-property-value val page-uuids)))
+                                    identity)]
+    (->> (map (fn [[prop-name val]]
+                [(get-ident all-idents prop-name)
+                 (translate-property-values val)])
+              properties)
+         (into {}))))
 
 (defn- create-page-uuids
   "Creates maps of unique page names, block contents and property names to their uuids. Used to
@@ -108,7 +113,10 @@
                    (let [property-map {:db/ident k
                                        :logseq.property/type built-in-type}]
                      [property-map v])
-                   (when-let [built-in-type' (get (:build/properties-ref-types new-block) built-in-type)]
+                   (when-let [built-in-type' (get (or (:build/properties-ref-types new-block)
+                                                      ;; Reasonable default for properties like logseq.property/default-value
+                                                      {:entity :number})
+                                                  built-in-type)]
                      (let [property-map {:db/ident k
                                          :logseq.property/type built-in-type'}]
                        [property-map v])))
@@ -122,7 +130,7 @@
                      [property-map v])))))
        (db-property-build/build-property-values-tx-m new-block)))
 
-(defn- extract-content-refs
+(defn- extract-basic-content-refs
   "Extracts basic refs from :block/title like `[[foo]]` or `[[UUID]]`. Can't
   use db-content/get-matched-ids because of named ref support.  Adding more ref
   support would require parsing each block with mldoc and extracting with
@@ -134,7 +142,7 @@
     (map second (re-seq page-ref/page-ref-re s))))
 
 (defn- ->block-tx [{:keys [build/properties] :as m} page-uuids all-idents page-id
-                   {properties-config :properties :keys [build-existing-tx?]}]
+                   {properties-config :properties :keys [build-existing-tx? extract-content-refs?] :as options}]
   (let [build-existing-tx?' (and build-existing-tx? (::existing-block? (meta m)) (not (:build/keep-uuid? m)))
         block (if build-existing-tx?'
                 (select-keys m [:block/uuid])
@@ -143,7 +151,7 @@
                  :block/order (db-order/gen-key nil)
                  :block/parent (or (:block/parent m) {:db/id page-id})})
         pvalue-tx-m (->property-value-tx-m block properties properties-config all-idents)
-        ref-strings (extract-content-refs (:block/title m))]
+        ref-strings (when extract-content-refs? (extract-basic-content-refs (:block/title m)))]
     (cond-> []
       ;; Place property values first since they are referenced by block
       (seq pvalue-tx-m)
@@ -153,7 +161,7 @@
                    (dissoc m :build/properties :build/tags :build/keep-uuid?)
                    (when (seq properties)
                      (->block-properties (merge properties (db-property-build/build-properties-with-ref-values pvalue-tx-m))
-                                         page-uuids all-idents))
+                                         page-uuids all-idents options))
                    (when-let [tags (:build/tags m)]
                      {:block/tags (mapv #(hash-map :db/ident (get-ident all-idents %))
                                         tags)})
@@ -171,7 +179,7 @@
                         :block/refs block-refs})))))))
 
 (defn- build-property-tx
-  [properties page-uuids all-idents property-db-ids
+  [properties page-uuids all-idents property-db-ids options
    [prop-name {:build/keys [property-classes] :as prop-m}]]
   (let [[new-block & additional-tx]
         (if-let [closed-values (seq (map #(merge {:uuid (random-uuid)} %) (:build/closed-values prop-m)))]
@@ -199,9 +207,10 @@
       true
       (conj
        (merge
-        new-block
+        (dissoc new-block :build/properties-ref-types)
         (when-let [props (not-empty (:build/properties prop-m))]
-          (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m)) page-uuids all-idents))
+          (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m))
+                              page-uuids all-idents options))
         (when (seq property-classes)
           {:logseq.property/classes
            (mapv #(hash-map :db/ident (get-ident all-idents %))
@@ -209,7 +218,7 @@
       true
       (into additional-tx))))
 
-(defn- build-properties-tx [properties page-uuids all-idents {:keys [build-existing-tx?]}]
+(defn- build-properties-tx [properties page-uuids all-idents {:keys [build-existing-tx?] :as options}]
   (let [properties' (if build-existing-tx?
                       (->> properties
                            (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/keep-uuid? v)))))
@@ -219,11 +228,11 @@
                              (map #(vector % (new-db-id)))
                              (into {}))
         new-properties-tx (vec
-                           (mapcat (partial build-property-tx properties' page-uuids all-idents property-db-ids)
+                           (mapcat (partial build-property-tx properties' page-uuids all-idents property-db-ids options)
                                    properties'))]
     new-properties-tx))
 
-(defn- build-classes-tx [classes properties-config uuid-maps all-idents {:keys [build-existing-tx?]}]
+(defn- build-classes-tx [classes properties-config uuid-maps all-idents {:keys [build-existing-tx?] :as options}]
   (let [classes' (if build-existing-tx?
                    (->> classes
                         (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/keep-uuid? v)))))
@@ -255,7 +264,8 @@
                              new-block
                              (dissoc class-m :build/properties :build/class-parent :build/class-properties :build/keep-uuid?)
                              (when-let [props (not-empty (:build/properties class-m))]
-                               (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m)) uuid-maps all-idents))
+                               (->block-properties (merge props (db-property-build/build-properties-with-ref-values pvalue-tx-m))
+                                                   uuid-maps all-idents options))
                              (when class-parent
                                {:logseq.property/parent
                                 (or (class-db-ids class-parent)
@@ -331,9 +341,11 @@
    [:graph-namespace {:optional true} :keyword]
    [:page-id-fn {:optional true} :any]
    [:auto-create-ontology? {:optional true} :boolean]
-   [:build-existing-tx? {:optional true} :boolean]])
+   [:build-existing-tx? {:optional true} :boolean]
+   [:extract-content-refs? {:optional true} :boolean]
+   [:translate-property-values? {:optional true} :boolean]])
 
-(defn- get-used-properties-from-options
+(defn get-used-properties-from-options
   "Extracts all used properties as a map of properties to their property values. Looks at properties
    from :build/properties and :build/class-properties. Properties from :build/class-properties have
    a ::no-value value"
@@ -388,7 +400,7 @@
             "Class and property db-idents have no overlap")
     all-idents))
 
-(defn- build-page-tx [page all-idents page-uuids properties]
+(defn- build-page-tx [page all-idents page-uuids properties options]
   (let [page' (dissoc page :build/tags :build/properties :build/keep-uuid?)
         pvalue-tx-m (->property-value-tx-m page' (:build/properties page) properties all-idents)]
     (cond-> []
@@ -401,8 +413,7 @@
          page'
          (when (seq (:build/properties page))
            (->block-properties (merge (:build/properties page) (db-property-build/build-properties-with-ref-values pvalue-tx-m))
-                               page-uuids
-                               all-idents))
+                               page-uuids all-idents options))
          (when-let [tag-idents (->> (:build/tags page) (map #(get-ident all-idents %)) seq)]
            {:block/tags (cond-> (mapv #(hash-map :db/ident %) tag-idents)
                           (empty? (set/intersection (set tag-idents) db-class/page-classes))
@@ -411,7 +422,7 @@
 (defn- build-pages-and-blocks-tx
   [pages-and-blocks all-idents page-uuids {:keys [page-id-fn properties build-existing-tx?]
                                            :or {page-id-fn :db/id}
-                                           :as opts}]
+                                           :as options}]
   (vec
    (mapcat
     (fn [{:keys [page blocks]}]
@@ -432,11 +443,11 @@
          ;; page tx
          (if build-existing-tx?'
            [(select-keys page [:block/uuid :block/created-at :block/updated-at])]
-           (build-page-tx page' all-idents page-uuids properties))
+           (build-page-tx page' all-idents page-uuids properties options))
          ;; blocks tx
          (reduce (fn [acc m]
                    (into acc
-                         (->block-tx m page-uuids all-idents (page-id-fn' page') opts)))
+                         (->block-tx m page-uuids all-idents (page-id-fn' page') options)))
                  []
                  blocks))))
     pages-and-blocks)))
@@ -475,7 +486,7 @@
              (mapcat
               (fn [{:keys [blocks]}]
                 (->> blocks
-                     (mapcat #(extract-content-refs (:block/title %)))
+                     (mapcat #(extract-basic-content-refs (:block/title %)))
                      (remove common-util/uuid-string?)
                      (remove existing-pages))))
              distinct
@@ -524,7 +535,7 @@
 
 (defn- pre-build-pages-and-blocks
   "Pre builds :pages-and-blocks before any indexes like page-uuids are made"
-  [pages-and-blocks properties]
+  [pages-and-blocks properties {:keys [:extract-content-refs?]}]
   (let [ensure-page-uuids (fn [m]
                             (if (get-in m [:page :block/uuid])
                               m
@@ -546,16 +557,20 @@
                                                    (or (:block/uuid page) (common-uuid/gen-uuid :journal-page-uuid date-int))
                                                    :block/tags :logseq.class/Journal})
                                            (with-meta {::new-page? (not (:block/uuid page))})))))
-                           m))]
-    ;; Order matters as some steps depend on previous step having prepared blocks or pages in a certain way
-    (->> pages-and-blocks
-         (add-new-pages-from-properties properties)
-         (map expand-journal)
-         (map expand-block-children)
-         add-new-pages-from-refs
-         ;; This needs to be last to ensure page metadata
-         (map ensure-page-uuids)
-         vec)))
+                           m))
+        ;; Order matters as some steps depend on previous step having prepared blocks or pages in a certain way
+        pages (->> pages-and-blocks
+                   (add-new-pages-from-properties properties)
+                   (map expand-journal)
+                   (map expand-block-children))]
+    (cond->> pages
+      extract-content-refs?
+      add-new-pages-from-refs
+      true
+      ;; This needs to be last to ensure page metadata
+      (map ensure-page-uuids)
+      true
+      vec)))
 
 (defn- infer-property-schema
   "Infers a property schema given a collection of its a property pair values"
@@ -597,10 +612,27 @@
     ;; (when (seq new-classes) (prn :new-classes new-classes))
     {:classes classes' :properties properties'}))
 
+(defn- get-possible-referenced-uuids
+  "Gets all possible ref uuids from either [:block/uuid X] or {:build/journal X}. Uuid scraping
+   is aggressive so some uuids may not be referenced"
+  [input-map]
+  (let [uuids (atom #{})
+        _ (walk/postwalk (fn [f]
+                           ;; This does get a few uuids that aren't :build/keep-uuid? but
+                           ;; that's ok because it consistently gets pvalue uuids
+                           (when (and (vector? f) (= :block/uuid (first f)))
+                             (swap! uuids conj (second f)))
+                           ;; All journals that don't have uuid and could be referenced
+                           (when (and (map? f) (:build/journal f) (not (:block/uuid f)))
+                             (swap! uuids conj (common-uuid/gen-uuid :journal-page-uuid (:build/journal f))))
+                           f)
+                         input-map)]
+    @uuids))
+
 (defn- build-blocks-tx*
   [{:keys [pages-and-blocks properties graph-namespace auto-create-ontology?]
     :as options}]
-  (let [pages-and-blocks' (pre-build-pages-and-blocks pages-and-blocks properties)
+  (let [pages-and-blocks' (pre-build-pages-and-blocks pages-and-blocks properties (dissoc options :pages-and-blocks :properties))
         page-uuids (create-page-uuids pages-and-blocks')
         {:keys [classes properties]} (if auto-create-ontology? (auto-create-ontology options) options)
         all-idents (create-all-idents properties classes graph-namespace)
@@ -628,7 +660,9 @@
       (:build-existing-tx? options)
       (update :init-tx
               (fn [init-tx]
-                (let [indices (mapv #(select-keys % [:block/uuid]) (filter :block/uuid init-tx))]
+                (let [indices
+                      (mapv #(hash-map :block/uuid %)
+                            (get-possible-referenced-uuids {:classes classes :properties properties :pages-and-blocks pages-and-blocks}))]
                   (into indices init-tx)))))))
 
 ;; Public API
@@ -645,6 +679,16 @@
                   (mapcat #(apply-to-block-and-all-children % f) children))))]
     (mapcat #(apply-to-block-and-all-children % f) blocks)))
 
+(defn update-each-block
+  "Calls fn f on each block including all children under :build/children"
+  [blocks f]
+  (mapv (fn [m]
+          (let [updated-m (f m)]
+            (if (:build/children m)
+              (assoc updated-m :build/children (update-each-block (:build/children m) f))
+              updated-m)))
+        blocks))
+
 (defn validate-options
   [{:keys [properties] :as options}]
   (when-let [errors (->> options (m/explain Options) me/humanize)]
@@ -693,7 +737,8 @@
      * :build/closed-values - Define closed values with a vec of maps. A map contains keys :uuid, :value and :icon.
      * :build/property-classes - Vec of class name keywords. Defines a property's range classes
      * :build/properties-ref-types - Map of internal ref types to public ref types that are valid only for this property.
-       Useful when remapping value ref types e.g. for :logseq.property/default-value
+       Useful when remapping value ref types e.g. for :logseq.property/default-value.
+       Default is `{:entity :number}`
      * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
    * :classes - This is a map to configure classes where the keys are class name keywords
      and the values are maps of datascript attributes e.g. `{:block/title \"Foo\"}`.
@@ -709,6 +754,11 @@
      existing in DB and are skipped for creation. This is useful for building tx on existing DBs e.g. for importing.
      Blocks and pages are updated with any attributes passed to it while all other node types are ignored for update
      unless :build/keep-uuid? is set.
+  * :extract-content-refs? - When set to true, plain text refs e.g. `[[foo]]` are automatically extracted to create pages
+    and to create refs in blocks. This is useful for testing but since it only partially works, not useful for exporting.
+    Default is true
+  * :translate-property-values? - When set to true, property values support special interpretation e.g. `[:build/page ..]`.
+    Default is true
   * :page-id-fn - custom fn that returns ent lookup id for page refs e.g. `[:block/uuid X]`
     Default is :db/id
 
@@ -718,9 +768,10 @@
    supported: :default, :url, :checkbox, :number, :node and :date. :checkbox and
    :number values are written as booleans and integers/floats. :node references
    are written as vectors e.g. `[:build/page {:block/title \"PAGE NAME\"}]`"
-  [options]
-  (validate-options options)
-  (build-blocks-tx* options))
+  [options*]
+  (let [options (merge {:extract-content-refs? true :translate-property-values? true} options*)]
+    (validate-options options)
+    (build-blocks-tx* options)))
 
 (defn create-blocks
   "Builds txs with build-blocks-tx and transacts them. Also provides a shorthand

deps/db/src/logseq/db/sqlite/export.cljs

@@ -2,6 +2,7 @@
   "Builds sqlite.build EDN to represent nodes in a graph-agnostic way.
    Useful for exporting and importing across DB graphs"
   (:require [clojure.set :as set]
+            [clojure.string :as string]
             [clojure.walk :as walk]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
@@ -10,7 +11,6 @@
             [logseq.db.frontend.content :as db-content]
             [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.build :as sqlite-build]))
 
@@ -39,18 +39,18 @@
 
 (defn- buildable-property-value-entity
   "Converts property value to a buildable version"
-  [property-ent pvalue]
-  (cond (ldb/internal-page? pvalue)
+  [property-ent pvalue {:keys [property-value-uuids?]}]
+  (cond (and (not property-value-uuids?) (ldb/internal-page? pvalue))
         ;; Should page properties be pulled here?
         [:build/page (cond-> (shallow-copy-page pvalue)
                        (seq (:block/tags pvalue))
                        (assoc :build/tags (->build-tags (:block/tags pvalue))))]
-        (entity-util/journal? pvalue)
+        (and (not property-value-uuids?) (entity-util/journal? pvalue))
         [:build/page {:build/journal (:block/journal-day pvalue)}]
         :else
-        (if (= :node (:logseq.property/type property-ent))
-          ;; Internal idents take precedence over uuid because they are keep data graph-agnostic
-          (if (some-> pvalue :db/ident db-malli-schema/internal-ident?)
+        (if (contains? #{:node :date} (:logseq.property/type property-ent))
+          ;; Idents take precedence over uuid because they are keep data graph-agnostic
+          (if (:db/ident pvalue)
             (:db/ident pvalue)
             ;; Use metadata distinguish from block references that don't exist like closed values
             ^::existing-property-value? [:block/uuid (:block/uuid pvalue)])
@@ -62,7 +62,7 @@
 (defn- buildable-properties
   "Originally copied from db-test/readable-properties. Modified so that property values are
    valid sqlite.build EDN"
-  [db ent-properties properties-config]
+  [db ent-properties properties-config options]
   (->> ent-properties
        (map (fn [[k v]]
               [k
@@ -74,17 +74,17 @@
                    (throw (ex-info (str "No closed value found for content: " (pr-str (db-property/property-value-content v))) {:properties properties-config})))
                  (cond
                    (de/entity? v)
-                   (buildable-property-value-entity (d/entity db k) v)
+                   (buildable-property-value-entity (d/entity db k) v options)
                    (and (set? v) (every? de/entity? v))
                    (let [property-ent (d/entity db k)]
-                     (set (map (partial buildable-property-value-entity property-ent) v)))
+                     (set (map #(buildable-property-value-entity property-ent % options) v)))
                    :else
                    v))]))
        (into {})))
 
 (defn- build-export-properties
   "The caller of this fn is responsible for building :build/:property-classes unless shallow-copy?"
-  [db user-property-idents {:keys [include-properties? include-uuid? shallow-copy?]}]
+  [db user-property-idents {:keys [include-properties? include-timestamps? include-uuid? shallow-copy?] :as options}]
   (let [properties-config-by-ent
         (->> user-property-idents
              (map (fn [ident]
@@ -93,14 +93,17 @@
                       [property
                        (cond-> (select-keys property
                                             (-> (disj db-property/schema-properties :logseq.property/classes)
-                                                (conj :block/title)))
+                                                (into [:block/title :block/collapsed?])))
                          include-uuid?
-                         (assoc :block/uuid (:block/uuid property))
+                         (assoc :block/uuid (:block/uuid property) :build/keep-uuid? true)
+                         include-timestamps?
+                         (merge (select-keys property [:block/created-at :block/updated-at]))
                          (and (not shallow-copy?) (:logseq.property/classes property))
                          (assoc :build/property-classes (mapv :db/ident (:logseq.property/classes property)))
                          (seq closed-values)
                          (assoc :build/closed-values
-                                (mapv #(cond-> {:value (db-property/property-value-content %) :uuid (random-uuid)}
+                                (mapv #(cond-> {:value (db-property/property-value-content %)
+                                                :uuid (:block/uuid %)}
                                          (:logseq.property/icon %)
                                          (assoc :icon (:logseq.property/icon %)))
                                       closed-values)))])))
@@ -112,25 +115,30 @@
     (if include-properties?
       (->> properties-config-by-ent
            (map (fn [[ent build-property]]
-                  (let [ent-properties (apply dissoc (db-property/properties ent) :block/tags db-property/schema-properties)]
+                  (let [ent-properties (apply dissoc (db-property/properties ent)
+                                              (into db-property/schema-properties db-property/public-db-attribute-properties))]
                     [(:db/ident ent)
                      (cond-> build-property
                        (seq ent-properties)
-                       (assoc :build/properties (buildable-properties db ent-properties properties-config)))])))
+                       (assoc :build/properties (buildable-properties db ent-properties properties-config options)))])))
            (into {}))
       properties-config)))
 
 (defn- build-export-class
   "The caller of this fn is responsible for building any classes or properties from this fn
    unless shallow-copy?"
-  [class-ent {:keys [include-parents? include-uuid? shallow-copy?]
+  [class-ent {:keys [include-parents? include-uuid? shallow-copy? include-timestamps?]
               :or {include-parents? true}}]
-  (cond-> (select-keys class-ent [:block/title])
+  (cond-> (select-keys class-ent [:block/title :block/collapsed?])
     include-uuid?
-    (assoc :block/uuid (:block/uuid class-ent))
+    (assoc :block/uuid (:block/uuid class-ent) :build/keep-uuid? true)
+    include-timestamps?
+    (merge (select-keys class-ent [:block/created-at :block/updated-at]))
     (and (:logseq.property.class/properties class-ent) (not shallow-copy?))
     (assoc :build/class-properties
            (mapv :db/ident (:logseq.property.class/properties class-ent)))
+    (and (not shallow-copy?) (:block/alias class-ent))
+    (assoc :block/alias (set (map #(vector :block/uuid (:block/uuid %)) (:block/alias class-ent))))
     ;; It's caller's responsibility to ensure parent is included in final export
     (and include-parents?
          (not shallow-copy?)
@@ -167,7 +175,7 @@
           (into {})))))
 
 (defn- build-node-properties
-  [db entity ent-properties properties]
+  [db entity ent-properties {:keys [properties] :as options}]
   (let [new-user-property-ids (->> (keys ent-properties)
                                    (concat (->> (:block/tags entity)
                                                 (mapcat :logseq.property.class/properties)
@@ -175,28 +183,35 @@
                                    ;; Built-in properties and any possible modifications are not exported
                                    (remove db-property/logseq-property?)
                                    (remove #(get properties %)))]
-    ;; Classes from hare are built in build-node-classes
-    (build-export-properties db new-user-property-ids {})))
+    ;; Classes from here are built in build-node-classes
+    (build-export-properties db new-user-property-ids options)))
 
 (defn- build-node-export
   "Given a block/page entity and optional existing properties, build an export map of its
    tags and properties"
-  [db entity {:keys [properties include-uuid-fn shallow-copy?]
-              :or {include-uuid-fn (constantly false)}}]
-  (let [ent-properties (dissoc (db-property/properties entity) :block/tags)
+  [db entity {:keys [properties include-uuid-fn shallow-copy? include-timestamps? exclude-ontology?]
+              :or {include-uuid-fn (constantly false)}
+              :as options}]
+  (let [ent-properties (apply dissoc (db-property/properties entity) db-property/public-db-attribute-properties)
         build-tags (when (seq (:block/tags entity)) (->build-tags (:block/tags entity)))
-        new-properties (when-not shallow-copy? (build-node-properties db entity ent-properties properties))
+        new-properties (when-not (or shallow-copy? exclude-ontology?)
+                         (build-node-properties db entity ent-properties (dissoc options :shallow-copy? :include-uuid-fn)))
         build-node (cond-> {:block/title (block-title entity)}
+                     (some? (:block/collapsed? entity))
+                     (assoc :block/collapsed? (:block/collapsed? entity))
                      (:block/link entity)
                      (assoc :block/link [:block/uuid (:block/uuid (:block/link entity))])
                      (include-uuid-fn (:block/uuid entity))
                      (assoc :block/uuid (:block/uuid entity) :build/keep-uuid? true)
+                     include-timestamps?
+                     (merge (select-keys entity [:block/created-at :block/updated-at]))
                      (and (not shallow-copy?) (seq build-tags))
                      (assoc :build/tags build-tags)
                      (and (not shallow-copy?) (seq ent-properties))
                      (assoc :build/properties
-                            (buildable-properties db ent-properties (merge properties new-properties))))
-        new-classes (when-not shallow-copy? (build-node-classes db build-node (:block/tags entity) new-properties))]
+                            (buildable-properties db ent-properties (merge properties new-properties) options)))
+        new-classes (when-not (or shallow-copy? exclude-ontology?)
+                      (build-node-classes db build-node (:block/tags entity) new-properties))]
     (cond-> {:node build-node}
       (seq new-classes)
       (assoc :classes new-classes)
@@ -215,7 +230,10 @@
                        (if (set? val-or-vals) val-or-vals [val-or-vals]))))
        set))
 
-(defn- merge-export-maps [& export-maps]
+(defn- merge-export-maps
+  "Merge export maps for partial graph exports. *Do not* use for a full graph
+  export because it makes assumptions about page identity"
+  [& export-maps]
   (let [pages-and-blocks
         (->> (mapcat :pages-and-blocks export-maps)
              ;; TODO: Group by more correct identity for title, same as check-for-existing-entities
@@ -297,15 +315,16 @@
   "Given a vec of block entities, returns the blocks in a sqlite.build EDN format
    and all properties and classes used in these blocks"
   [db blocks {:keys [include-children?] :or {include-children? true} :as opts}]
-  (let [*properties (atom {})
-        *classes (atom {})
+  (let [*properties (atom (or (get-in opts [:graph-ontology :properties]) {}))
+        *classes (atom (or (get-in opts [:graph-ontology :classes]) {}))
         *pvalue-uuids (atom #{})
         id-map (into {} (map (juxt :db/id identity)) blocks)
         children (if include-children? (group-by #(get-in % [:block/parent :db/id]) blocks) {})
         build-block (fn build-block [block*]
                       (let [child-nodes (mapv build-block (get children (:db/id block*) []))
                             {:keys [node properties classes]}
-                            (build-node-export db block* (assoc opts :properties @*properties))
+                            (build-node-export db block* (-> (dissoc opts :graph-ontology)
+                                                             (assoc :properties @*properties)))
                             new-pvalue-uuids (get-pvalue-uuids node)]
                         (when (seq properties) (swap! *properties merge properties))
                         (when (seq classes) (swap! *classes merge classes))
@@ -314,10 +333,12 @@
                           (seq child-nodes) (assoc :build/children child-nodes))))
         roots (remove #(contains? id-map (get-in % [:block/parent :db/id])) blocks)
         exported-blocks (mapv build-block roots)]
-    {:blocks exported-blocks
-     :properties @*properties
-     :classes @*classes
-     :pvalue-uuids @*pvalue-uuids}))
+    (cond-> {:blocks exported-blocks
+             :pvalue-uuids @*pvalue-uuids}
+      (not= @*properties (get-in opts [:graph-ontology :properties]))
+      (assoc :properties @*properties)
+      (not= @*classes (get-in opts [:graph-ontology :classes]))
+      (assoc :classes @*classes))))
 
 (defn- build-uuid-block-export [db pvalue-uuids content-ref-ents {:keys [page-entity]}]
   (let [content-ref-blocks (set (remove entity-util/page? content-ref-ents))
@@ -364,32 +385,51 @@
     (merge {::block (:node node-export)}
            block-export)))
 
-(defn- build-page-blocks-export [db page-entity {:keys [properties classes blocks]}]
-  (let [page-ent-export (build-node-export db page-entity {:properties properties})
+(defn- build-page-blocks-export [db page-entity {:keys [properties classes blocks] :as options}]
+  (let [options' (cond-> (dissoc options :classes :blocks :graph-ontology)
+                   (:exclude-ontology? options)
+                   (assoc :properties (get-in options [:graph-ontology :properties])))
+        page-ent-export (build-node-export db page-entity options')
+        page-pvalue-uuids (get-pvalue-uuids (:node page-ent-export))
         page (merge (dissoc (:node page-ent-export) :block/title)
-                    (shallow-copy-page page-entity))
+                    (shallow-copy-page page-entity)
+                    (when (:block/alias page-entity)
+                      {:block/alias (set (map #(vector :block/uuid (:block/uuid %)) (:block/alias page-entity)))}))
         page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}]
                             :properties properties
                             :classes classes}]
-    (merge-export-maps page-blocks-export page-ent-export)))
+    (assoc (merge-export-maps page-blocks-export page-ent-export)
+           :pvalue-uuids page-pvalue-uuids)))
 
-(defn- build-page-export
-  "Exports page for given page eid"
-  [db eid]
+(defn- get-page-blocks [db eid]
+  (->> (d/datoms db :avet :block/page eid)
+       (map :e)
+       (map #(d/entity db %))))
+
+(defn- build-page-export*
+  [db eid page-blocks* options]
   (let [page-entity (d/entity db eid)
-        datoms (d/datoms db :avet :block/page eid)
-        block-eids (mapv :e datoms)
-        page-blocks* (map #(d/entity db %) block-eids)
-        {:keys [content-ref-uuids content-ref-ents] :as content-ref-export} (build-content-ref-export db page-blocks*)
         page-blocks (->> page-blocks*
                          (sort-by :block/order)
                          ;; Remove property value blocks as they are exported in a block's :build/properties
                          (remove #(:logseq.property/created-from-property %)))
         {:keys [pvalue-uuids] :as blocks-export}
-        (build-blocks-export db page-blocks {:include-uuid-fn content-ref-uuids})
+        (build-blocks-export db page-blocks options)
+        page-blocks-export (build-page-blocks-export db page-entity (merge blocks-export options))
+        page-block-uuids (set/union pvalue-uuids (:pvalue-uuids page-blocks-export))
+        page-export (assoc page-blocks-export :pvalue-uuids page-block-uuids)]
+    page-export))
+
+(defn- build-page-export
+  "Exports page for given page eid"
+  [db eid]
+  (let [page-blocks* (get-page-blocks db eid)
+        {:keys [content-ref-ents] :as content-ref-export} (build-content-ref-export db page-blocks*)
+        {:keys [pvalue-uuids] :as page-export*}
+        (build-page-export* db eid page-blocks* {:include-uuid-fn (:content-ref-uuids content-ref-export)})
+        page-entity (d/entity db eid)
         uuid-block-export (build-uuid-block-export db pvalue-uuids content-ref-ents {:page-entity page-entity})
-        page-blocks-export (build-page-blocks-export db page-entity blocks-export)
-        page-export (finalize-export-maps db page-blocks-export uuid-block-export content-ref-export)]
+        page-export (finalize-export-maps db page-export* uuid-block-export content-ref-export)]
     page-export))
 
 (defn build-view-nodes-export* [db nodes opts]
@@ -420,7 +460,7 @@
   "Exports given nodes from a view. Nodes are a random mix of blocks and pages"
   [db eids]
   (let [nodes (map #(d/entity db %) eids)
-        property-value-ents (mapcat #(->> (dissoc (db-property/properties %) :block/tags)
+        property-value-ents (mapcat #(->> (apply dissoc (db-property/properties %) db-property/public-db-attribute-properties)
                                           vals
                                           (filter de/entity?))
                                     nodes)
@@ -434,26 +474,36 @@
 
 (defn- build-graph-ontology-export
   "Exports a graph's tags and properties"
-  [db]
-  (let [user-property-idents (d/q '[:find [?db-ident ...]
+  [db {:keys [exclude-namespaces] :as options}]
+  (let [exclude-regex (when (seq exclude-namespaces)
+                        (re-pattern (str "^("
+                                         (string/join "|" (map name exclude-namespaces))
+                                         ")(\\.|$)")))
+        user-property-idents (d/q '[:find [?db-ident ...]
                                     :where [?p :db/ident ?db-ident]
                                     [?p :block/tags :logseq.class/Property]
                                     (not [?p :logseq.property/built-in?])]
                                   db)
-        properties (build-export-properties db user-property-idents {:include-properties? true})
+        user-property-idents' (if (seq exclude-namespaces)
+                                (remove #(re-find exclude-regex (namespace %)) user-property-idents)
+                                user-property-idents)
+        properties (build-export-properties db user-property-idents' (merge options {:include-properties? true}))
         class-ents (->> (d/q '[:find [?class ...]
                                :where [?class :block/tags :logseq.class/Tag]
                                (not [?class :logseq.property/built-in?])]
                              db)
-                        (map #(d/entity db %)))
+                        (map #(d/entity db %))
+                        (remove #(and (seq exclude-namespaces) (re-find exclude-regex (namespace (:db/ident %))))))
         classes
         (->> class-ents
              (map (fn [ent]
-                    (let [ent-properties (dissoc (db-property/properties ent) :block/tags :logseq.property/parent)]
+                    (let [ent-properties (apply dissoc (db-property/properties ent) :logseq.property/parent db-property/public-db-attribute-properties)]
                       (vector (:db/ident ent)
-                              (cond-> (build-export-class ent {})
+                              (cond-> (build-export-class ent options)
                                 (seq ent-properties)
-                                (assoc :build/properties (buildable-properties db ent-properties properties)))))))
+                                (assoc :build/properties
+                                       (-> (buildable-properties db ent-properties properties options)
+                                           (dissoc :logseq.property.class/properties))))))))
              (into {}))]
     (cond-> {}
       (seq properties)
@@ -461,6 +511,136 @@
       (seq classes)
       (assoc :classes classes))))
 
+(defn- get-graph-content-ref-uuids
+  [db {:keys [:exclude-built-in-pages?]}]
+  (let [;; Add support for exclude-built-in-pages? and block-titles as needed
+        block-titles (map :v (d/datoms db :avet :block/title))
+        block-links (if exclude-built-in-pages?
+                      (->> (d/datoms db :avet :block/link)
+                           (keep #(when-not (:logseq.property/built-in? (:block/page (d/entity db (:e %))))
+                                    (:block/uuid (d/entity db (:v %))))))
+                      (->> (d/datoms db :avet :block/link)
+                           (map #(:block/uuid (d/entity db (:v %))))))
+        content-ref-uuids (concat (->> block-titles
+                                       (filter string?)
+                                       (mapcat db-content/get-matched-ids))
+                                  block-links)]
+    (set content-ref-uuids)))
+
+(defn- build-graph-pages-export
+  "Handles pages, journals and their blocks"
+  [db graph-ontology options*]
+  (let [options (merge options*
+                       {:graph-ontology graph-ontology}
+                       ;; dont exclude when ontology is incomplete because :closed values can fail so have to build ontology
+                       (when (empty? (:exclude-namespaces options*))
+                         {:exclude-ontology? true}))
+        page-ids (concat (map :e (d/datoms db :avet :block/tags :logseq.class/Page))
+                         (map :e (d/datoms db :avet :block/tags :logseq.class/Journal)))
+        page-exports (mapv (fn [eid]
+                             (let [page-blocks* (get-page-blocks db eid)]
+                               (build-page-export* db eid page-blocks* (merge options {:include-uuid-fn (constantly true)}))))
+                           page-ids)
+        page-exports' (remove (fn [page-export]
+                                (and (:exclude-built-in-pages? options)
+                                     (get-in page-export [:pages-and-blocks 0 :page :build/properties :logseq.property/built-in?])))
+                              page-exports)
+        alias-uuids  (mapcat (fn [{:keys [pages-and-blocks]}]
+                               (mapcat #(when-let [aliases (get-in % [:page :block/alias])]
+                                          (map second aliases))
+                                       pages-and-blocks))
+                             page-exports')
+        pages-export {:pages-and-blocks (vec (mapcat :pages-and-blocks page-exports'))
+                      :pvalue-uuids (into (set (mapcat :pvalue-uuids page-exports'))
+                                          alias-uuids)}]
+    pages-export))
+
+(defn- build-graph-files
+  [db {:keys [include-timestamps?]}]
+  (->> (d/q '[:find [(pull ?b [:file/path :file/content :file/created-at :file/last-modified-at]) ...]
+              :where [?b :file/path]] db)
+       (mapv #(if include-timestamps?
+                (select-keys % [:file/path :file/content :file/created-at :file/last-modified-at])
+                (select-keys % [:file/path :file/content])))))
+
+(defn- build-kv-values
+  [db]
+  (->> (d/q '[:find [(pull ?b [:db/ident :kv/value]) ...]
+              :where [?b :kv/value]] db)
+       ;; Don't export schema-version as frontend sets this and shouldn't be overridden
+       (remove #(= :logseq.kv/schema-version (:db/ident %)))
+       vec))
+
+(defn remove-uuids-if-not-ref [export-map all-ref-uuids]
+  (let [remove-uuid-if-not-ref (fn [m] (if (contains? all-ref-uuids (:block/uuid m))
+                                         m
+                                         (dissoc m :block/uuid :build/keep-uuid?)))]
+    (-> export-map
+        (update :classes update-vals remove-uuid-if-not-ref)
+        (update :properties update-vals remove-uuid-if-not-ref)
+        (update :pages-and-blocks
+                (fn [pages-and-blocks]
+                  (mapv (fn [{:keys [page blocks]}]
+                          {:page (remove-uuid-if-not-ref page)
+                           :blocks (sqlite-build/update-each-block blocks remove-uuid-if-not-ref)})
+                        pages-and-blocks))))))
+
+(defn sort-pages-and-blocks
+  "Provide a reliable sort order since this tends to be large. Helps with diffing
+   and readability"
+  [pages-and-blocks]
+  (vec
+   (sort-by #(or (get-in % [:page :block/title]) (str (get-in % [:page :build/journal])))
+            pages-and-blocks)))
+
+(defn- add-ontology-for-include-namespaces
+  "Adds :properties to export for given namespace parents. Current use case is for :exclude-namespaces
+   so no need to add :classes yet"
+  [db {::keys [auto-include-namespaces] :as graph-export}]
+  (let [include-regex (re-pattern (str "^("
+                                       (string/join "|" (map name auto-include-namespaces))
+                                       ")(\\.|$)"))
+        used-properties
+        (->> (sqlite-build/get-used-properties-from-options graph-export)
+             keys
+             (remove db-property/logseq-property?)
+             (filter #(re-find include-regex (namespace %)))
+             (map #(vector % (select-keys (d/entity db %) [:logseq.property/type :db/cardinality])))
+             (into {}))]
+    (-> (merge-export-maps (select-keys graph-export [:properties])
+                           {:properties used-properties})
+        (select-keys [:properties]))))
+
+(defn- build-graph-export
+  "Exports whole graph. Has the following options:
+   * :include-timestamps? - When set timestamps are included on all blocks
+   * :exclude-namespaces - A set of parent namespaces to exclude from properties and classes.
+     This is useful for graphs seeded with an ontology e.g. schema.org as it eliminates noisy and needless
+     export+import
+   * :exclude-built-in-pages? - When set built-in pages are excluded from export"
+  [db options*]
+  (let [options (merge options* {:property-value-uuids? true})
+        content-ref-uuids (get-graph-content-ref-uuids db options)
+        ontology-options (merge options {:include-uuid? true})
+        ontology-export (build-graph-ontology-export db ontology-options)
+        ontology-pvalue-uuids (set (concat (mapcat get-pvalue-uuids (vals (:properties ontology-export)))
+                                           (mapcat get-pvalue-uuids (vals (:classes ontology-export)))))
+        pages-export (build-graph-pages-export db ontology-export options)
+        graph-export* (-> (merge ontology-export pages-export) (dissoc :pvalue-uuids))
+        graph-export (if (seq (:exclude-namespaces options))
+                       (assoc graph-export* ::auto-include-namespaces (:exclude-namespaces options))
+                       graph-export*)
+        all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export))
+        files (build-graph-files db options)
+        kv-values (build-kv-values db)
+        ;; Remove all non-ref uuids after all nodes are built.
+        ;; Only way to ensure all pvalue uuids present across block types
+        graph-export' (-> (remove-uuids-if-not-ref graph-export all-ref-uuids)
+                          (update :pages-and-blocks sort-pages-and-blocks))]
+    (merge graph-export'
+           {::graph-files files
+            ::kv-values kv-values})))
+
 (defn- find-undefined-classes-and-properties [{:keys [classes properties pages-and-blocks]}]
   (let [referenced-classes
         (->> (concat (mapcat :build/property-classes (vals properties))
@@ -492,7 +672,8 @@
                      (mapcat #(sqlite-build/extract-from-blocks (:blocks %) (fn [m] (some-> m :block/uuid vector)))
                              pages-and-blocks))
              set)
-        ;; only looks one-level deep in properties e.g. not inside :build/page
+        ;; Only looks one-level deep in properties e.g. not inside :build/page
+        ;; Doesn't find :block/link refs
         ref-uuids
         (->> (concat (mapcat get-pvalue-uuids (vals classes))
                      (mapcat get-pvalue-uuids (vals properties))
@@ -503,11 +684,14 @@
 
 (defn- ensure-export-is-valid
   "Checks that export map is usable by sqlite.build including checking that
-   all referenced properties and classes are defined"
-  [export-map]
-  (sqlite-build/validate-options export-map)
+   all referenced properties and classes are defined. Checks related to properties and
+   classes are disabled when :exclude-namespaces is set because those checks can't be done"
+  [export-map {:keys [graph-options]}]
+  (when-not (seq (:exclude-namespaces graph-options)) (sqlite-build/validate-options export-map))
   (let [undefined-uuids (find-undefined-uuids export-map)
-        undefined (cond-> (find-undefined-classes-and-properties export-map)
+        undefined (cond-> {}
+                    (empty? (:exclude-namespaces graph-options))
+                    (merge (find-undefined-classes-and-properties export-map))
                     (seq undefined-uuids)
                     (assoc :uuids undefined-uuids))]
     (when (seq undefined)
@@ -526,9 +710,11 @@
           :view-nodes
           (build-view-nodes-export db (:node-ids options))
           :graph-ontology
-          (build-graph-ontology-export db))]
-    (ensure-export-is-valid (dissoc export-map ::block))
-    export-map))
+          (build-graph-ontology-export db {})
+          :graph
+          (build-graph-export db (:graph-options options)))]
+    (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values) options)
+    (assoc export-map ::export-type export-type)))
 
 ;; Import fns
 ;; ==========
@@ -548,9 +734,10 @@
 (defn- check-for-existing-entities
   "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import.
    Also checks for property conflicts between existing properties and properties to be imported"
-  [db {:keys [pages-and-blocks classes properties]} property-conflicts]
+  [db {:keys [pages-and-blocks classes properties] ::keys [export-type] :as export-map} property-conflicts]
   (let [export-map
-        (cond-> {:build-existing-tx? true}
+        (cond-> {:build-existing-tx? true
+                 :extract-content-refs? false}
           (seq pages-and-blocks)
           (assoc :pages-and-blocks
                  (mapv (fn [m]
@@ -578,13 +765,20 @@
                                            :expected (select-keys ent [:logseq.property/type :db/cardinality])}))
                                  [k (assoc v :block/uuid (:block/uuid ent))])
                                [k v])))
-                      (into {}))))
-        export-map'
-        (walk/postwalk (fn [f]
-                         (if (and (vector? f) (= :build/page (first f)))
-                           [:build/page (add-uuid-to-page-if-exists db (second f))]
-                           f))
-                       export-map)]
+                      (into {})))
+          ;; Graph export doesn't use :build/page so this speeds up build
+          (= :graph export-type)
+          (assoc :translate-property-values? false)
+          (= :graph export-type)
+          ;; Currently all graph-files are created by app so no need to distinguish between user and built-in ones yet
+          (merge (dissoc export-map :pages-and-blocks :classes :properties)))
+        export-map' (if (= :graph export-type)
+                      export-map
+                      (walk/postwalk (fn [f]
+                                       (if (and (vector? f) (= :build/page (first f)))
+                                         [:build/page (add-uuid-to-page-if-exists db (second f))]
+                                         f))
+                                     export-map))]
     export-map'))
 
 (defn- build-block-import-options
@@ -599,16 +793,36 @@
     (merge-export-maps export-map {:pages-and-blocks pages-and-blocks})))
 
 (defn build-import
-  "Given an entity's export map, build the import tx to create it"
+  "Given an entity's export map, build the import tx to create it. In addition to standard sqlite.build keys,
+   an export map can have the following namespaced keys:
+   * ::export-type - Keyword indicating export type
+   * ::block - Block map for a :block export
+   * ::graph-files - Vec of files for a :graph export
+   * ::kv-values - Vec of :kv/value maps for a :graph export
+   * ::auto-include-namespaces - A set of parent namespaces to include from properties and classes
+     for a :graph export. See :exclude-namespaces in build-graph-export for a similar option
+
+   This fn then returns a map of txs to transact with the following keys:
+   * :init-tx - Txs that must be transacted first, usually because they define new properties
+   * :block-props-tx - Txs to transact after :init-tx, usually because they use newly defined properties
+   * :misc-tx - Txs to transact unrelated to other txs"
   [export-map* db {:keys [current-block]}]
   (let [export-map (if (and (::block export-map*) current-block)
                      (build-block-import-options current-block export-map*)
                      export-map*)
+        export-map' (if (and (= :graph (::export-type export-map*)) (seq (::auto-include-namespaces export-map*)))
+                      (merge (dissoc export-map :properties ::auto-include-namespaces)
+                             (add-ontology-for-include-namespaces db export-map))
+                      export-map)
         property-conflicts (atom [])
-        export-map' (check-for-existing-entities db export-map property-conflicts)]
+        export-map'' (check-for-existing-entities db export-map' property-conflicts)]
     (if (seq @property-conflicts)
       (do
         (js/console.error :property-conflicts @property-conflicts)
         {:error (str "The following imported properties conflict with the current graph: "
                      (pr-str (mapv :property-id @property-conflicts)))})
-      (sqlite-build/build-blocks-tx export-map'))))
+      (if (= :graph (::export-type export-map''))
+        (-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files ::kv-values ::export-type))
+            (assoc :misc-tx (vec (concat (::graph-files export-map'')
+                                         (::kv-values export-map'')))))
+        (sqlite-build/build-blocks-tx export-map'')))))

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -1,9 +1,14 @@
 (ns logseq.db.sqlite.export-test
   (:require [cljs.pprint]
             [cljs.test :refer [deftest is testing]]
+            [clojure.walk :as walk]
             [datascript.core :as d]
+            [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
+            [logseq.common.uuid :as common-uuid]
+            [logseq.db :as ldb]
             [logseq.db.frontend.validate :as db-validate]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.test.helper :as db-test]
@@ -35,6 +40,54 @@
     (sqlite-export/build-export @import-conn {:export-type :block
                                               :block-id (:db/id import-block)})))
 
+(defn- export-page-and-import-to-another-graph
+  "Exports given page from one graph/conn, imports it to a 2nd graph, validates
+  it and then exports the page from the 2nd graph"
+  [export-conn import-conn page-title]
+  (let [page (db-test/find-page-by-title @export-conn page-title)
+        {:keys [init-tx block-props-tx] :as _txs}
+        (-> (sqlite-export/build-export @export-conn {:export-type :page :page-id (:db/id page)})
+            ;; ((fn [x] (cljs.pprint/pprint {:export x}) x))
+            (sqlite-export/build-import @import-conn {}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)
+        _ (validate-db @import-conn)
+        page2 (db-test/find-page-by-title @import-conn page-title)]
+    (sqlite-export/build-export @import-conn {:export-type :page :page-id (:db/id page2)})))
+
+(defn- import-second-time-assertions [conn conn2 page-title original-data
+                                      & {:keys [transform-expected-blocks]
+                                         :or {transform-expected-blocks (fn [bs] (into bs bs))}}]
+  (let [page (db-test/find-page-by-title @conn2 page-title)
+        imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
+        updated-page (db-test/find-page-by-title @conn2 page-title)
+        expected-page-and-blocks
+        (update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks)]
+
+    ;; Assume first page is one being imported for now
+    (is (= (first expected-page-and-blocks)
+           (first (:pages-and-blocks imported-page)))
+        "Blocks are appended to existing page")
+    (is (= (:block/created-at page) (:block/created-at updated-page))
+        "Existing page didn't get re-created")
+    (is (= (:block/updated-at page) (:block/updated-at updated-page))
+        "Existing page didn't get updated")))
+
+(defn- export-graph-and-import-to-another-graph
+  "Exports graph and imports it to a 2nd graph, validates it and then exports the 2nd graph"
+  [export-conn import-conn export-options]
+  (let [{:keys [init-tx block-props-tx misc-tx] :as _txs}
+        (-> (sqlite-export/build-export @export-conn {:export-type :graph :graph-options export-options})
+            (sqlite-export/build-import @import-conn {}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)
+        _ (d/transact! import-conn misc-tx)
+        _ (validate-db @import-conn)
+        imported-graph (sqlite-export/build-export @import-conn {:export-type :graph :graph-options export-options})]
+    imported-graph))
+
 (defn- expand-properties
   "Add default values to properties of an input export map to test against a
   db-based export map"
@@ -61,6 +114,8 @@
                  (assoc :block/title (name k)))]))
        (into {})))
 
+(def sort-pages-and-blocks sqlite-export/sort-pages-and-blocks)
+
 ;; Tests
 ;; =====
 
@@ -173,40 +228,6 @@
            (first (:pages-and-blocks imported-block)))
         "Imported page equals exported page of page ref")))
 
-(defn- export-page-and-import-to-another-graph
-  "Exports given page from one graph/conn, imports it to a 2nd graph, validates
-  it and then exports the page from the 2nd graph"
-  [export-conn import-conn page-title]
-  (let [page (db-test/find-page-by-title @export-conn page-title)
-        {:keys [init-tx block-props-tx] :as _txs}
-        (-> (sqlite-export/build-export @export-conn {:export-type :page :page-id (:db/id page)})
-            ;; ((fn [x] (cljs.pprint/pprint {:export x}) x))
-            (sqlite-export/build-import @import-conn {}))
-        ;; _ (cljs.pprint/pprint _txs)
-        _ (d/transact! import-conn init-tx)
-        _ (d/transact! import-conn block-props-tx)
-        _ (validate-db @import-conn)
-        page2 (db-test/find-page-by-title @import-conn page-title)]
-    (sqlite-export/build-export @import-conn {:export-type :page :page-id (:db/id page2)})))
-
-(defn- import-second-time-assertions [conn conn2 page-title original-data
-                                      & {:keys [transform-expected-blocks]
-                                         :or {transform-expected-blocks (fn [bs] (into bs bs))}}]
-  (let [page (db-test/find-page-by-title @conn2 page-title)
-        imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
-        updated-page (db-test/find-page-by-title @conn2 page-title)
-        expected-page-and-blocks
-        (update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks)]
-
-    ;; Assume first page is one being imported for now
-    (is (= (first expected-page-and-blocks)
-           (first (:pages-and-blocks imported-page)))
-        "Blocks are appended to existing page")
-    (is (= (:block/created-at page) (:block/created-at updated-page))
-        "Existing page didn't get re-created")
-    (is (= (:block/updated-at page) (:block/updated-at updated-page))
-        "Existing page didn't get updated")))
-
 ;; Tests a variety of blocks including block children with new properties, blocks with users classes
 ;; and blocks with built-in properties and classes
 (deftest import-page-with-different-blocks
@@ -262,9 +283,11 @@
                        :build/property-classes [:user.class/NodeClass]}
                       :user.property/p2
                       {:logseq.property/type :default}}
+         :extract-content-refs? false
          :pages-and-blocks
          [{:page {:block/title "page1"}
            :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}
+                    {:block/title (str "not a page ref `" (page-ref/->page-ref "foo") "`")}
                     {:block/title (str "block ref to " (page-ref/->page-ref block-uuid))}
                     {:block/title "ref in properties"
                      :build/properties {:user.property/p2 (str "pvalue ref to " (page-ref/->page-ref pvalue-page-uuid))}}
@@ -506,4 +529,191 @@
 
     (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-nodes)))
     (is (= (expand-properties (:properties original-data)) (:properties imported-nodes)))
-    (is (= (expand-classes (:classes original-data)) (:classes imported-nodes)))))
+    (is (= (expand-classes (:classes original-data)) (:classes imported-nodes)))))
+
+(defn- build-original-graph-data
+  [& {:keys [exclude-namespaces?]}]
+  (let [internal-block-uuid (random-uuid)
+        favorited-uuid (random-uuid)
+        block-pvalue-uuid (random-uuid)
+        property-pvalue-uuid (random-uuid)
+        page-pvalue-uuid (random-uuid)
+        page-object-uuid (random-uuid)
+        page-alias-uuid (random-uuid)
+        closed-value-uuid (random-uuid)
+        property-uuid (random-uuid)
+        class-uuid (random-uuid)
+        journal-uuid (common-uuid/gen-uuid :journal-page-uuid 19650201)
+        original-data
+        {:properties
+         {:user.property/num {:logseq.property/type :number
+                              :block/uuid property-uuid
+                              :build/keep-uuid? true
+                              :build/properties (if exclude-namespaces?
+                                                  {}
+                                                  {:user.property/node #{[:block/uuid property-pvalue-uuid]}
+                                                   :logseq.property/default-value 42})}
+          :user.property/default-closed
+          {:logseq.property/type :default
+           :build/closed-values [{:value "joy" :uuid closed-value-uuid}
+                                 {:value "sad" :uuid (random-uuid)}]}
+          :user.property/checkbox {:logseq.property/type :checkbox}
+          :user.property/date {:logseq.property/type :date}
+          :user.property/url {:logseq.property/type :url
+                              :build/properties {:logseq.property/description "desc for url"}}
+          :user.property/node {:logseq.property/type :node
+                               :db/cardinality :db.cardinality/many
+                               :build/property-classes [:user.class/MyClass]}}
+         :classes
+         {:user.class/MyClass {:build/properties {:user.property/url "https://example.com/MyClass"}
+                               :block/uuid class-uuid
+                               :build/keep-uuid? true}
+          :user.class/MyClass2 {:build/class-parent :user.class/MyClass
+                                :build/properties {:logseq.property/description "tests child class"}}}
+         :pages-and-blocks
+         [{:page {:block/title "page1"
+                  :block/uuid favorited-uuid :build/keep-uuid? true
+                  :build/properties {:user.property/checkbox false
+                                     :user.property/node #{[:block/uuid page-pvalue-uuid]}}}
+           :blocks [{:block/title "b1"
+                     :build/properties {:user.property/num 1
+                                        :user.property/default-closed [:block/uuid closed-value-uuid]
+                                        :user.property/date [:block/uuid journal-uuid]}}
+                    {:block/title "b2" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}}
+                    {:block/title "b3" :build/properties {:user.property/node #{[:block/uuid page-object-uuid]}}}]}
+          {:page {:block/title "page object"
+                  :block/uuid page-object-uuid
+                  :build/keep-uuid? true}
+           :blocks []}
+          {:page {:block/title "page2" :build/tags [:user.class/MyClass2]}
+           :blocks [{:block/title "hola" :block/uuid internal-block-uuid :build/keep-uuid? true}
+                    {:block/title "myclass object 1"
+                     :build/tags [:user.class/MyClass]
+                     :block/uuid block-pvalue-uuid
+                     :build/keep-uuid? true}
+                    (cond-> {:block/title "myclass object 2"
+                             :build/tags [:user.class/MyClass]}
+                      (not exclude-namespaces?)
+                      (merge {:block/uuid property-pvalue-uuid
+                              :build/keep-uuid? true}))
+                    {:block/title "myclass object 3"
+                     :build/tags [:user.class/MyClass]
+                     :block/uuid page-pvalue-uuid
+                     :build/keep-uuid? true}
+                    {:block/title "ref blocks"
+                     :build/children
+                     [{:block/title (str "internal block ref to " (page-ref/->page-ref internal-block-uuid))}
+                      {:block/title "node block"
+                       :build/properties {:user.property/node #{[:block/uuid block-pvalue-uuid]}}}
+                      {:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}
+                      {:block/title (str "class ref to " (page-ref/->page-ref class-uuid))}]}]}
+          {:page {:block/title "Alias for 2/28" :block/uuid page-alias-uuid :build/keep-uuid? true}
+           :blocks []}
+          {:page {:build/journal 20250228
+                  :block/alias #{[:block/uuid page-alias-uuid]}
+                  :build/properties {:user.property/num 1}}
+           :blocks [{:block/title "journal block"}]}
+          {:page {:build/journal 19650201
+                  :block/uuid journal-uuid
+                  :build/keep-uuid? true}
+           :blocks []}
+          ;; built-in pages
+          {:page {:block/title "Contents" :build/properties {:logseq.property/built-in? true}}
+           :blocks [{:block/title "right sidebar"}]}
+          {:page {:block/title common-config/favorites-page-name
+                  :build/properties {:logseq.property/built-in? true, :logseq.property/hide? true}}
+           :blocks [(ldb/build-favorite-tx favorited-uuid)]}
+          {:page {:block/title common-config/views-page-name
+                  :build/properties {:logseq.property/built-in? true, :logseq.property/hide? true}}
+           :blocks [{:block/title "All"
+                     :build/properties {:logseq.property/view-for :logseq.class/Task
+                                        :logseq.property.view/feature-type :class-objects}}
+                    {:block/title "All"
+                     :build/properties {:logseq.property/view-for :user.class/MyClass
+                                        :logseq.property.view/feature-type :class-objects}}
+                    {:block/title "Linked references",
+                     :build/properties
+                     {:logseq.property.view/type :logseq.property.view/type.list,
+                      :logseq.property.view/feature-type :linked-references,
+                      :logseq.property/view-for [:block/uuid journal-uuid]}}]}]
+         ::sqlite-export/graph-files
+         [{:file/path "logseq/config.edn"
+           :file/content "{:foo :bar}"}
+          {:file/path "logseq/custom.css"
+           :file/content ".foo {background-color: blue}"}
+          {:file/path "logseq/custom.js"
+           :file/content "// comment"}]}]
+    original-data))
+
+(deftest import-graph
+  (let [original-data (build-original-graph-data)
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        ;; set to an unobtainable version to test this ident
+        _ (d/transact! conn [{:db/ident :logseq.kv/schema-version :kv/value {:major 1 :minor 0}}])
+        original-kv-values (remove #(= :logseq.kv/schema-version (:db/ident %))
+                                   (d/q '[:find [(pull ?b [:db/ident :kv/value]) ...] :where [?b :kv/value]] @conn))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn)
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {})]
+
+    ;; (cljs.pprint/pprint (set (:pages-and-blocks original-data)))
+    ;; (cljs.pprint/pprint (set (:pages-and-blocks imported-graph)))
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (sort-pages-and-blocks (:pages-and-blocks original-data))
+    ;;                                                 (:pages-and-blocks imported-graph))))
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= 1 (count (d/datoms @conn2 :avet :block/title "page object")))
+        "No duplicate pages for pvalue uuids used more than once")
+    (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
+    (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")
+    (is (= original-kv-values (::sqlite-export/kv-values imported-graph))
+        "All :kv/value entities are imported except for ignored ones")
+    (is (not= (:kv/value (d/entity @conn :logseq.kv/schema-version))
+              (:kv/value (d/entity @conn2 :logseq.kv/schema-version)))
+        "Ignored :kv/value is not updated")))
+
+(deftest import-graph-with-timestamps
+  (let [original-data* (build-original-graph-data)
+        original-data (-> original-data*
+                          (update :pages-and-blocks
+                                  (fn [pages-and-blocks]
+                                    (walk/postwalk (fn [e]
+                                                     (if (and (map? e) (or (:block/title e) (:build/journal e)))
+                                                       (common-util/block-with-timestamps e)
+                                                       e))
+                                                   pages-and-blocks)))
+                          (update :classes update-vals common-util/block-with-timestamps)
+                          (update :properties update-vals common-util/block-with-timestamps)
+                          (update ::sqlite-export/graph-files
+                                  (fn [files]
+                                    (mapv #(let [now (js/Date.)]
+                                             (merge % {:file/created-at now :file/last-modified-at now}))
+                                          files))))
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn)
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {:include-timestamps? true})]
+
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (sort-pages-and-blocks (:pages-and-blocks original-data))
+    ;;                                                 (:pages-and-blocks imported-graph))))
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
+    (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")))
+
+(deftest import-graph-with-exclude-namespaces
+  (let [original-data (build-original-graph-data {:exclude-namespaces? true})
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn-with-blocks
+               {:properties (update-vals (:properties original-data) #(dissoc % :build/properties))
+                :classes (update-vals (:classes original-data) #(dissoc % :build/properties))})
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {:exclude-namespaces #{:user}})]
+
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (sort-pages-and-blocks (:pages-and-blocks original-data))
+    ;;                                                 (:pages-and-blocks imported-graph))))
+    (is (= (sort-pages-and-blocks (:pages-and-blocks original-data)) (:pages-and-blocks imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")))

scripts/src/logseq/tasks/db_graph/create_graph_with_properties.cljs

@@ -10,6 +10,7 @@
             [clojure.edn :as edn]
             [clojure.set :as set]
             [clojure.string :as string]
+            [cljs.pprint :as pprint]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
             [logseq.common.util.date-time :as date-time-util]
@@ -190,6 +191,8 @@
   "Options spec"
   {:help {:alias :h
           :desc "Print help"}
+   :file {:alias :f
+          :desc "File to save generated sqlite.build EDN"}
    :config {:alias :c
             :coerce edn/read-string
             :desc "EDN map to add to config.edn"}})
@@ -209,7 +212,9 @@
             (fse/removeSync db-path))
         conn (outliner-cli/init-conn dir db-name {:additional-config (:config options)
                                                   :classpath (cp/get-classpath)})
-        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx (create-init-data))
+        init-data (create-init-data)
+        _ (when (:file options) (fs/writeFileSync (:file options) (with-out-str (pprint/pprint init-data))))
+        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx init-data)
         existing-names (set (map :v (d/datoms @conn :avet :block/title)))
         conflicting-names (set/intersection existing-names (set (keep :block/title init-tx)))]
     (when (seq conflicting-names)

scripts/src/logseq/tasks/dev/db_and_file_graphs.clj

@@ -118,8 +118,8 @@
                        (map str)
                        (into [;; e.g. block/properties :title
                               "block/properties :"
-                              ;; anything org mode except for org.babashka
-                              "org[^\\.]"
+                              ;; anything org mode except for org.babashka or urls like schema.org
+                              "[^\\.]org[^\\.]"
                               "#+BEGIN_"
                               "#+END_"
                               "pre-block"]))

src/main/frontend/components/container.cljs

@@ -1072,6 +1072,7 @@
       [:a#download-as-json-v2.hidden]
       [:a#download-as-transit-debug.hidden]
       [:a#download-as-sqlite-db.hidden]
+      [:a#download-as-db-edn.hidden]
       [:a#download-as-roam-json.hidden]
       [:a#download-as-html.hidden]
       [:a#download-as-zip.hidden]

src/main/frontend/components/export.cljs

@@ -4,6 +4,7 @@
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
+            [frontend.handler.db-based.export :as db-export-handler]
             [frontend.handler.export :as export]
             [frontend.handler.export.html :as export-html]
             [frontend.handler.export.opml :as export-opml]
@@ -100,6 +101,10 @@
           [:div
            [:a.font-medium {:on-click #(export/export-repo-as-zip! current-repo)}
             (t :export-zip)]])
+        (when db-based?
+          [:div
+           [:a.font-medium {:on-click #(db-export-handler/export-repo-as-db-edn! current-repo)}
+            (t :export-db-edn)]])
         (when db-based?
           [:div
            [:a.font-medium {:on-click #(export/export-repo-as-debug-transit! current-repo)}

src/main/frontend/components/imports.cljs

@@ -12,6 +12,7 @@
             [frontend.fs :as fs]
             [frontend.handler.file-based.import :as file-import-handler]
             [frontend.handler.db-based.editor :as db-editor-handler]
+            [frontend.handler.db-based.import :as db-import-handler]
             [frontend.handler.import :as import-handler]
             [frontend.handler.notification :as notification]
             [frontend.handler.repo :as repo-handler]
@@ -70,7 +71,7 @@
                           :error))))
 
 (defn- lsq-import-handler
-  [e & {:keys [sqlite? debug-transit? graph-name]}]
+  [e & {:keys [sqlite? debug-transit? graph-name db-edn?]}]
   (let [file      (first (array-seq (.-files (.-target e))))
         file-name (some-> (gobj/get file "name")
                           (string/lower-case))
@@ -91,7 +92,7 @@
             (set! (.-onload reader)
                   (fn []
                     (let [buffer (.-result ^js reader)]
-                      (import-handler/import-from-sqlite-db! buffer graph-name finished-cb)
+                      (db-import-handler/import-from-sqlite-db! buffer graph-name finished-cb)
                       (shui/dialog-close!))))
             (set! (.-onerror reader) (fn [e] (js/console.error e)))
             (set! (.-onabort reader) (fn [e]
@@ -99,7 +100,7 @@
                                        (js/console.error e)))
             (.readAsArrayBuffer reader file))))
 
-      debug-transit?
+      (or debug-transit? db-edn?)
       (let [graph-name (string/trim graph-name)]
         (cond
           (string/blank? graph-name)
@@ -112,7 +113,9 @@
           (do
             (state/set-state! :graph/importing :logseq)
             (let [reader (js/FileReader.)
-                  import-f import-handler/import-from-debug-transit!]
+                  import-f (if db-edn?
+                             db-import-handler/import-from-edn-file!
+                             db-import-handler/import-from-debug-transit!)]
               (set! (.-onload reader)
                     (fn [e]
                       (let [text (.. e -target -result)]
@@ -121,7 +124,9 @@
                          text
                          #(do
                             (state/set-state! :graph/importing nil)
-                            (finished-cb))))))
+                            (finished-cb)
+                            ;; graph input not closing
+                            (shui/dialog-close-all!))))))
               (.readAsText reader file)))))
 
       (or edn? json?)
@@ -449,7 +454,7 @@
    [importing?])
   [:<>])
 
-(rum/defc importer < rum/reactive
+(rum/defc ^:large-vars/cleanup-todo importer < rum/reactive
   [{:keys [query-params]}]
   (let [support-file-based? (config/local-file-based-graph? (state/get-current-repo))
         importing? (state/sub :graph/importing)]
@@ -497,8 +502,6 @@
              [:span.flex.flex-col
               [[:strong "Debug Transit"]
                [:small "Import debug transit file into a new DB graph"]]]
-             ;; Test form style changes
-             #_[:a.button {:on-click #(import-file-to-db-handler nil {:import-graph-fn js/alert})} "Open"]
              [:input.absolute.hidden
               {:id "import-debug-transit"
                :type "file"
@@ -506,11 +509,24 @@
                             (shui/dialog-open!
                              #(set-graph-name-dialog e {:debug-transit? true})))}]])
 
+          (when (or (util/electron?) util/web-platform?)
+            [:label.action-input.flex.items-center.mx-2.my-2
+             [:span.as-flex-center [:i (svg/logo 28)]]
+             [:span.flex.flex-col
+              [[:strong "EDN to DB graph"]
+               [:small "Import a DB graph's EDN export into a new DB graph"]]]
+             [:input.absolute.hidden
+              {:id "import-db-edn"
+               :type "file"
+               :on-change (fn [e]
+                            (shui/dialog-open!
+                             #(set-graph-name-dialog e {:db-edn? true})))}]])
+
           (when (and (util/electron?) support-file-based?)
             [:label.action-input.flex.items-center.mx-2.my-2
              [:span.as-flex-center [:i (svg/logo 28)]]
              [:span.flex.flex-col
-              [[:strong "EDN / JSON"]
+              [[:strong "EDN / JSON to plain text graph"]
                [:small (t :on-boarding/importing-lsq-desc)]]]
              [:input.absolute.hidden
               {:id "import-lsq"

src/main/frontend/components/repo.cljs

@@ -241,7 +241,9 @@
                                                        (not (and rtc-graph? remote?)))
                                                 (state/pub-event! [:graph/open-new-window url])
                                                 (cond
-                                                  (:root graph) ; exists locally
+                                                  ;; exists locally?
+                                                  (or (:root graph)
+                                                      (and db-only? (not rtc-graph?)))
                                                   (state/pub-event! [:graph/switch url])
 
                                                   (and rtc-graph? remote?)

src/main/frontend/handler/db_based/export.cljs

@@ -1,16 +1,14 @@
 (ns frontend.handler.db-based.export
   "Handles DB graph exports and imports across graphs"
   (:require [cljs.pprint :as pprint]
-            [clojure.edn :as edn]
-            [frontend.db :as db]
+            [clojure.string :as string]
+            [frontend.config :as config]
             [frontend.handler.notification :as notification]
-            [frontend.handler.ui :as ui-handler]
             [frontend.state :as state]
             [frontend.util :as util]
             [frontend.util.page :as page-util]
+            [goog.dom :as gdom]
             [logseq.db :as ldb]
-            [logseq.db.sqlite.export :as sqlite-export]
-            [logseq.shui.ui :as shui]
             [promesa.core :as p]))
 
 (defn ^:export export-block-data []
@@ -61,56 +59,32 @@
                            (count (:properties result)) " properties"))
       (notification/show! "Copied graphs's ontology data!" :success))))
 
-(defn- import-submit [import-inputs _e]
-  (let [export-map (try (edn/read-string (:import-data @import-inputs)) (catch :default _err ::invalid-import))
-        import-block? (::sqlite-export/block export-map)
-        block (when import-block?
-                (if-let [eid (:block-id (first (state/get-editor-args)))]
-                  (db/entity [:block/uuid eid])
-                  (notification/show! "No block found" :warning)))]
-    (if (= ::invalid-import export-map)
-      (notification/show! "The submitted EDN data is invalid! Fix and try again." :warning)
-      (let [{:keys [init-tx block-props-tx error] :as txs}
-            (try
-              (sqlite-export/build-import export-map
-                                          (db/get-db)
-                                          (when block {:current-block block}))
-              (catch :default e
-                (js/console.error "Import EDN error: " e)
-                {:error "An unexpected error occurred during import. See the javascript console for details."}))]
-        (pprint/pprint txs)
-        (if error
-          (notification/show! error :error)
-          (p/do
-            ;; TODO: Use metadata that supports undo
-            (db/transact! (state/get-current-repo) init-tx
-                          (if import-block? {:save-block true} {::sqlite-export/imported-data? true}))
-
-            (when (seq block-props-tx)
-              (db/transact! (state/get-current-repo) block-props-tx
-                            (if import-block? {:save-block true} {::sqlite-export/imported-data? true})))
+(defn- export-graph-edn-data []
+  (when-let [^Object worker @state/*db-worker]
+    (p/let [result* (.export-edn worker
+                                 (state/get-current-repo)
+                                 (ldb/write-transit-str {:export-type :graph
+                                                         :graph-options {:include-timestamps? true}}))
+            result (ldb/read-transit-str result*)
+            pull-data (with-out-str (pprint/pprint result))]
+      pull-data)))
 
-            (when-not import-block?
-              (state/clear-async-query-state!)
-              (ui-handler/re-render-root!)
-              (notification/show! "Import successful!" :success))))
-        ;; Also close cmd-k
-        (shui/dialog-close-all!)))))
+;; Copied from handler.export
+(defn- file-name [repo extension]
+  (-> (string/replace repo config/local-db-prefix "")
+      (string/replace #"^/+" "")
+      (str "_" (quot (util/time-ms) 1000))
+      (str "." (string/lower-case (name extension)))))
 
-(defn ^:export import-edn-data
-  []
-  (let [import-inputs (atom {:import-data "" :import-block? false})]
-    (shui/dialog-open!
-     [:div
-      [:label.flex.my-2.text-lg "Import EDN Data"]
-      #_[:label.block.flex.items-center.py-3
-         (shui/checkbox {:on-checked-change #(swap! import-inputs update :import-block? not)})
-         [:small.pl-2 (str "Import into current block")]]
-      (shui/textarea {:placeholder "{}"
-                      :class "overflow-y-auto"
-                      :rows 10
-                      :auto-focus true
-                      :on-change (fn [^js e] (swap! import-inputs assoc :import-data (util/evalue e)))})
-      (shui/button {:class "mt-3"
-                    :on-click (partial import-submit import-inputs)}
-                   "Import")])))
+(defn export-repo-as-db-edn!
+  [repo]
+  (p/let [edn-str (export-graph-edn-data)]
+    (when edn-str
+      (let [data-str (some->> edn-str
+                              js/encodeURIComponent
+                              (str "data:text/edn;charset=utf-8,"))
+            filename (file-name repo :edn)]
+        (when-let [anchor (gdom/getElement "download-as-db-edn")]
+          (.setAttribute anchor "href" data-str)
+          (.setAttribute anchor "download" filename)
+          (.click anchor))))))

src/main/frontend/handler/db_based/import.cljs

@@ -0,0 +1,154 @@
+(ns frontend.handler.db-based.import
+  "Handles DB graph imports"
+  (:require [clojure.edn :as edn]
+            [cljs.pprint :as pprint]
+            [frontend.config :as config]
+            [frontend.db :as db]
+            [frontend.handler.notification :as notification]
+            [frontend.handler.repo :as repo-handler]
+            [frontend.handler.ui :as ui-handler]
+            [frontend.persist-db :as persist-db]
+            [frontend.state :as state]
+            [frontend.util :as util]
+            [logseq.db :as ldb]
+            [logseq.db.sqlite.export :as sqlite-export]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.shui.ui :as shui]
+            [promesa.core :as p]))
+
+(defn import-from-sqlite-db!
+  [buffer bare-graph-name finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)]
+    (->
+     (p/do!
+      (persist-db/<import-db graph buffer)
+      (state/add-repo! {:url graph})
+      (repo-handler/restore-and-setup-repo! graph {:import-type :sqlite-db})
+      (state/set-current-repo! graph)
+      (persist-db/<export-db graph {})
+      (db/transact! graph (sqlite-util/import-tx :sqlite-db))
+      (finished-ok-handler))
+     (p/catch
+      (fn [e]
+        (js/console.error e)
+        (notification/show!
+         (str (.-message e))
+         :error))))))
+
+(defn import-from-debug-transit!
+  [bare-graph-name raw finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        datoms (ldb/read-transit-str raw)]
+    (p/do!
+     (persist-db/<new graph {:import-type :debug-transit
+                             :datoms datoms})
+     (state/add-repo! {:url graph})
+     (repo-handler/restore-and-setup-repo! graph {:import-type :debug-transit})
+     (db/transact! graph (sqlite-util/import-tx :debug-transit))
+     (state/set-current-repo! graph)
+     (finished-ok-handler))))
+
+(defn- safe-build-edn-import [export-map import-options]
+  (try
+    (sqlite-export/build-import export-map (db/get-db) import-options)
+    (catch :default e
+      (js/console.error "Import EDN error: " e)
+      {:error "An unexpected error occurred building the import. See the javascript console for details."})))
+
+(defn- import-edn-data-from-file
+  [export-map]
+  (let [{:keys [init-tx block-props-tx misc-tx error] :as _txs} (safe-build-edn-import export-map {})]
+    ;; (cljs.pprint/pprint _txs)
+    (if error
+      (notification/show! error :error)
+      (let [tx-meta {::sqlite-export/imported-data? true}
+            repo (state/get-current-repo)]
+        (p/do
+          (db/transact! repo init-tx tx-meta)
+          (when (seq block-props-tx)
+            (db/transact! repo block-props-tx tx-meta))
+          (when (seq misc-tx)
+            (db/transact! repo misc-tx tx-meta)))))))
+
+(defn import-from-edn-file!
+  "Creates a new DB graph and imports sqlite.build EDN file"
+  [bare-graph-name file-body finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        finished-error-handler
+        #(do
+           (state/set-state! :graph/importing nil)
+           (shui/dialog-close-all!))
+        edn-data (try
+                   (edn/read-string file-body)
+                   (catch :default e
+                     (js/console.error e)
+                     (notification/show! "The given EDN file is not valid EDN. Please fix and try again."
+                                         :error)
+                     (finished-error-handler)
+                     nil))]
+    (when (some? edn-data)
+      (-> (p/do!
+           (persist-db/<new graph {:import-type :edn})
+           (state/add-repo! {:url graph})
+           (repo-handler/restore-and-setup-repo! graph {:import-type :edn})
+           (state/set-current-repo! graph)
+           (import-edn-data-from-file edn-data)
+           (finished-ok-handler))
+          (p/catch
+           (fn [e]
+             (js/console.error e)
+             (notification/show! (str "Unexpected error: " (.-message e))
+                                 :error)
+             (finished-error-handler)))))))
+
+(defn- import-edn-data-from-form [import-inputs _e]
+  (let [export-map (try (edn/read-string (:import-data @import-inputs)) (catch :default _err ::invalid-import))
+        import-block? (::sqlite-export/block export-map)
+        block (when import-block?
+                (if-let [eid (:block-id (first (state/get-editor-args)))]
+                  (db/entity [:block/uuid eid])
+                  (notification/show! "No block found" :warning)))]
+    (if (= ::invalid-import export-map)
+      (notification/show! "The submitted EDN data is invalid! Please fix and try again." :warning)
+      (let [{:keys [init-tx block-props-tx misc-tx error] :as txs}
+            (safe-build-edn-import export-map (when block {:current-block block}))]
+        (pprint/pprint txs)
+        (if error
+          (notification/show! error :error)
+          ;; TODO: When not import-block, use metadata that supports undo
+          (let [tx-meta (if import-block? {:outliner-op :save-block} {::sqlite-export/imported-data? true})
+                repo (state/get-current-repo)]
+            (-> (p/do
+                  (db/transact! repo init-tx tx-meta)
+                  (when (seq block-props-tx)
+                    (db/transact! repo block-props-tx tx-meta))
+                  (when (seq misc-tx)
+                    (db/transact! repo misc-tx tx-meta))
+                  (when-not import-block?
+                    (state/clear-async-query-state!)
+                    (ui-handler/re-render-root!)
+                    (notification/show! "Import successful!" :success)))
+                (p/catch (fn [e]
+                           (js/console.error "Import EDN error: " e)
+                           (notification/show! "An unexpected error occurred during import. See the javascript console for details." :error))))))
+        ;; Also close cmd-k
+        (shui/dialog-close-all!)))))
+
+(defn ^:export import-edn-data-dialog
+  "Displays dialog which allows users to paste and import sqlite.build EDN Data"
+  []
+  (let [import-inputs (atom {:import-data "" :import-block? false})]
+    (shui/dialog-open!
+     [:div
+      [:label.flex.my-2.text-lg "Import EDN Data"]
+      #_[:label.block.flex.items-center.py-3
+         (shui/checkbox {:on-checked-change #(swap! import-inputs update :import-block? not)})
+         [:small.pl-2 (str "Import into current block")]]
+      (shui/textarea {:placeholder "{}"
+                      :class "overflow-y-auto"
+                      :rows 10
+                      :auto-focus true
+                      :on-change (fn [^js e] (swap! import-inputs assoc :import-data (util/evalue e)))})
+      (shui/button {:class "mt-3"
+                    :on-click (partial import-edn-data-from-form import-inputs)}
+                   "Import")])))

src/main/frontend/handler/export.cljs

@@ -12,8 +12,6 @@
    [frontend.extensions.zip :as zip]
    [frontend.external.roam-export :as roam-export]
    [frontend.handler.assets :as assets-handler]
-   ;; Loads commands
-   [frontend.handler.db-based.export]
    [frontend.handler.export.common :as export-common-handler]
    [frontend.handler.notification :as notification]
    [frontend.idb :as idb]

src/main/frontend/handler/import.cljs

@@ -5,7 +5,6 @@
             [clojure.edn :as edn]
             [clojure.string :as string]
             [clojure.walk :as walk]
-            [frontend.config :as config]
             [frontend.db :as db]
             [frontend.db.async :as db-async]
             [frontend.format.block :as block]
@@ -13,12 +12,8 @@
             [frontend.handler.editor :as editor]
             [frontend.handler.notification :as notification]
             [frontend.handler.page :as page-handler]
-            [frontend.handler.repo :as repo-handler]
-            [frontend.persist-db :as persist-db]
             [frontend.state :as state]
             [frontend.util :as util]
-            [logseq.db :as ldb]
-            [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.whiteboard :as gp-whiteboard]
             [medley.core :as medley]
@@ -165,25 +160,6 @@
                            form))]
      (walk/postwalk tree-trans-fn tree-vec))))
 
-(defn import-from-sqlite-db!
-  [buffer bare-graph-name finished-ok-handler]
-  (let [graph (str config/db-version-prefix bare-graph-name)]
-    (->
-     (p/do!
-      (persist-db/<import-db graph buffer)
-      (state/add-repo! {:url graph})
-      (repo-handler/restore-and-setup-repo! graph {:import-type :sqlite-db})
-      (state/set-current-repo! graph)
-      (persist-db/<export-db graph {})
-      (db/transact! graph (sqlite-util/import-tx :sqlite-db))
-      (finished-ok-handler))
-     (p/catch
-      (fn [e]
-        (js/console.error e)
-        (notification/show!
-         (str (.-message e))
-         :error))))))
-
 (defn import-from-edn!
   [raw finished-ok-handler]
   (try
@@ -230,16 +206,3 @@
     (async/go
       (async/<! (import-from-tree! clj-data tree-vec-translate-json))
       (finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated
-
-(defn import-from-debug-transit!
-  [bare-graph-name raw finished-ok-handler]
-  (let [graph (str config/db-version-prefix bare-graph-name)
-        datoms (ldb/read-transit-str raw)]
-    (p/do!
-     (persist-db/<new graph {:import-type :debug-transit
-                             :datoms datoms})
-     (state/add-repo! {:url graph})
-     (repo-handler/restore-and-setup-repo! graph {:import-type :debug-transit})
-     (db/transact! graph (sqlite-util/import-tx :debug-transit))
-     (state/set-current-repo! graph)
-     (finished-ok-handler nil))))

src/main/frontend/modules/shortcut/config.cljs

@@ -627,7 +627,7 @@
 
    :misc/import-edn-data {:binding []
                           :db-graph? true
-                          :fn :frontend.handler.db-based.export/import-edn-data}
+                          :fn :frontend.handler.db-based.import/import-edn-data-dialog}
 
    :dev/validate-db   {:binding []
                        :db-graph? true
@@ -647,8 +647,9 @@
   [keyword-fn]
   (fn []
     (if-let [resolved-fn (some-> (namespace keyword-fn)
-                                 ;; export is reserved word
+                                 ;; handle reserved words
                                  (string/replace-first ".export" ".export$")
+                                 (string/replace-first ".import" ".import$")
                                  find-ns-obj
                                  (aget (munge (name keyword-fn))))]
       (resolved-fn)

src/resources/dicts/en.edn

@@ -462,6 +462,7 @@
  :export-opml "Export as OPML"
  :export-public-pages "Export public pages"
  :export-json "Export as JSON"
+ :export-db-edn "Export EDN file"
  :export-sqlite-db "Export SQLite DB"
  :export-zip "Export both SQLite DB and assets"
  :export-roam-json "Export as Roam JSON"