enhance: block export supports content refs

Also refactor block tests and how export-maps are built and

deps/db/src/logseq/db/sqlite/export.cljs

@@ -177,14 +177,53 @@
                        (if (set? val-or-vals) val-or-vals [val-or-vals]))))
        set))
 
+(defn- merge-export-maps [& export-maps]
+  (let [pages-and-blocks (reduce into [] (keep :pages-and-blocks export-maps))
+        ;; Use merge-with to preserve new-property?
+        properties (apply merge-with merge (keep :properties export-maps))
+        classes (apply merge-with merge (keep :classes export-maps))]
+    (cond-> {:pages-and-blocks pages-and-blocks}
+      (seq properties)
+      (assoc :properties properties)
+      (seq classes)
+      (assoc :classes classes))))
+
+(defn- build-content-ref-export
+  "Builds an export config (and additional info) for refs in the given blocks. All the exported
+   entities found in block refs include their uuid in order to preserve the relationship to the blocks"
+  [db page-blocks]
+  (let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks))
+        content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
+        content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents)
+        content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
+                                 (update-vals (build-export-properties db prop-ids {:include-uuid? true})
+                                              #(merge % {:build/new-property? true})))
+        content-ref-classes (when-let [class-ents (seq (filter ldb/class? content-ref-ents))]
+                              (->> class-ents
+                                   ;; TODO: Export class parents when there's ability to control granularity of export
+                                   (map #(vector (:db/ident %)
+                                                 (assoc (build-export-class % {:include-parents? false :include-uuid? true})
+                                                        :build/new-class? true)))
+                                   (into {})))]
+    {:content-ref-uuids content-ref-uuids
+     :content-ref-ents content-ref-ents
+     :properties content-ref-properties
+     :classes content-ref-classes
+     :pages-and-blocks (mapv #(hash-map :page (assoc (shallow-copy-page %) :block/uuid (:block/uuid %)))
+                             content-ref-pages)}))
+
 (defn build-block-export
   [db eid]
-  (let [export-map (build-entity-export db (d/entity db eid) {})
-        pvalue-uuids (get-pvalue-uuids (:build/block export-map))]
+  (let [block-entity (d/entity db eid)
+        {:keys [content-ref-uuids _content-ref-ents] :as content-ref-export} (build-content-ref-export db [block-entity])
+        block-export* (build-entity-export db block-entity {:include-uuid-fn content-ref-uuids})
+        pvalue-uuids (get-pvalue-uuids (:build/block block-export*))
+        block-export (assoc (merge-export-maps block-export* content-ref-export)
+                            :build/block (:build/block block-export*))]
     ;; Maybe add support for this later
     (when (seq pvalue-uuids)
       (throw (ex-info "Exporting a block with :node block objects is not supported" {})))
-    export-map))
+    block-export))
 
 (defn- build-blocks-tree
   "Given a page's block entities, returns the blocks in a sqlite.build EDN format
@@ -230,30 +269,6 @@
      :classes (apply merge (map :classes uuid-block-pages))
      :pages-and-blocks (mapv #(select-keys % [:page :blocks]) uuid-block-pages)}))
 
-(defn- build-content-ref-export
-  "Builds an export config (and additional info) for refs in the given blocks. All the exported
-   entities found in block refs include their uuid in order to preserve the relationship to the blocks"
-  [db page-blocks]
-  (let [content-ref-uuids (set (mapcat (comp db-content/get-matched-ids block-title) page-blocks))
-        content-ref-ents (map #(d/entity db [:block/uuid %]) content-ref-uuids)
-        content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents)
-        content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
-                                 (update-vals (build-export-properties db prop-ids {:include-uuid? true})
-                                              #(merge % {:build/new-property? true})))
-        content-ref-classes (when-let [class-ents (seq (filter ldb/class? content-ref-ents))]
-                              (->> class-ents
-                                   ;; TODO: Export class parents when there's ability to control granularity of export
-                                   (map #(vector (:db/ident %)
-                                                 (assoc (build-export-class % {:include-parents? false :include-uuid? true})
-                                                        :build/new-class? true)))
-                                   (into {})))]
-    {:content-ref-uuids content-ref-uuids
-     :content-ref-ents content-ref-ents
-     :properties content-ref-properties
-     :classes content-ref-classes
-     :pages-and-blocks (mapv #(hash-map :page (assoc (shallow-copy-page %) :block/uuid (:block/uuid %)))
-                             content-ref-pages)}))
-
 (defn build-page-export [db eid]
   (let [page-entity (d/entity db eid)
         ;; TODO: Fetch unloaded page datoms
@@ -271,27 +286,10 @@
         page-ent-export (build-entity-export db page-entity {:properties properties})
         page (merge (dissoc (:build/block page-ent-export) :block/title)
                     (shallow-copy-page page-entity))
-        pages-and-blocks
-        (cond-> [{:page page :blocks blocks}]
-          (seq (:pages-and-blocks uuid-block-export))
-          (into (:pages-and-blocks uuid-block-export))
-          (seq (:pages-and-blocks content-ref-export))
-          (into (:pages-and-blocks content-ref-export)))
-        ;; Use merge-with to preserve new-property?
-        properties' (merge-with merge properties
-                                (:properties page-ent-export)
-                                (:properties content-ref-export)
-                                (:properties uuid-block-export))
-        classes' (merge-with merge classes
-                             (:classes page-ent-export)
-                             (:classes content-ref-export)
-                             (:classes uuid-block-export))
-        page-export
-        (cond-> {:pages-and-blocks pages-and-blocks}
-          (seq properties')
-          (assoc :properties properties')
-          (seq classes')
-          (assoc :classes classes'))]
+        page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}]
+                            :properties properties
+                            :classes classes}
+        page-export (merge-export-maps page-blocks-export page-ent-export uuid-block-export content-ref-export)]
     page-export))
 
 (defn build-graph-ontology-export
@@ -356,16 +354,13 @@
 (defn- build-block-import-options
   "Builds options for sqlite-build to import into current-block"
   [current-block export-map]
-  (let [{:build/keys [block]}
-        (merge-with merge
-                    export-map
-                    {:build/block
+  (let [block (merge (:build/block export-map)
                      {:block/uuid (:block/uuid current-block)
-                      :block/page (select-keys (:block/page current-block) [:block/uuid])}})
+                      :block/page (select-keys (:block/page current-block) [:block/uuid])})
         pages-and-blocks
         [{:page (select-keys (:block/page block) [:block/uuid])
           :blocks [(dissoc block :block/page)]}]]
-    (assoc export-map :pages-and-blocks pages-and-blocks)))
+    (merge-export-maps export-map {:pages-and-blocks pages-and-blocks})))
 
 (defn- build-page-import-options
   [db export-map]

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -2,107 +2,103 @@
   (:require [cljs.test :refer [deftest is testing]]
             [datascript.core :as d]
             [logseq.common.util.page-ref :as page-ref]
-            [logseq.db :as ldb]
-            [logseq.db.frontend.property :as db-property]
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.test.helper :as db-test]))
 
+(defn- export-block-and-import-to-another-block
+  "Exports given block from one graph/conn, imports it to a 2nd block and then
+   exports the 2nd block. The two blocks do not have to be in the same graph"
+  [export-conn import-conn export-block-content import-block-content]
+  (let [export-block (db-test/find-block-by-content @export-conn export-block-content)
+        import-block (db-test/find-block-by-content @import-conn import-block-content)
+        {:keys [init-tx block-props-tx] :as _txs}
+        (->> (sqlite-export/build-block-export @export-conn [:block/uuid (:block/uuid export-block)])
+             (sqlite-export/build-import @import-conn {:current-block import-block}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)]
+    (sqlite-export/build-block-export @import-conn (:db/id import-block))))
+
 (deftest import-block-in-same-graph
-  (let [conn (db-test/create-conn-with-blocks
-              {:properties {:default-many {:logseq.property/type :default :db/cardinality :many}}
-               :classes {:MyClass {:build/class-properties [:default-many]}}
-               :pages-and-blocks
-               [{:page {:block/title "page1"}
-                 :blocks [{:block/title "export"
-                           :build/properties {:default-many #{"foo" "bar" "baz"}}
-                           :build/tags [:MyClass]}
-                          {:block/title "import"}]}]})
-        export-block (db-test/find-block-by-content @conn "export")
-        import-block* (db-test/find-block-by-content @conn "import")
-        {:keys [init-tx block-props-tx]}
-        (->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)])
-             (sqlite-export/build-import @conn {:current-block import-block*}))
-        _ (assert (empty? block-props-tx) "This is empty for properties that already exist and thus no transacted")
-        _ (d/transact! conn init-tx)
-        import-block (d/entity @conn (:db/id import-block*))]
-    (is (= []
-           (filter #(or (:db/id %) (:db/ident %)) init-tx))
-        "Tx doesn't try to create new blocks or modify existing idents")
+  (let [original-data
+        {:properties {:user.property/default-many
+                      {:block/title "default-many" :logseq.property/type :default :db/cardinality :db.cardinality/many}}
+         :classes {:user.class/MyClass
+                   {:block/title "MyClass" :build/class-properties [:user.property/default-many]}}
+         :pages-and-blocks
+         [{:page {:block/title "page1"}
+           :blocks [{:block/title "export"
+                     :build/properties {:user.property/default-many #{"foo" "bar" "baz"}}
+                     :build/tags [:user.class/MyClass]}
+                    {:block/title "import"}]}]}
+        conn (db-test/create-conn-with-blocks original-data)
+        imported-block (export-block-and-import-to-another-block conn conn "export" "import")]
 
-    (is (= "export" (:block/title import-block))
-        "imported block title equals exported one")
-    (is (= {:user.property/default-many #{"foo" "bar" "baz"}
-            :block/tags [:user.class/MyClass]}
-           (db-test/readable-properties import-block))
-        "imported block properties and tags equals exported one")))
+    (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
+           (:build/block imported-block))
+        "Imported block equals exported block")
+    (is (= (:properties original-data) (:properties imported-block)))
+    (is (= (:classes original-data) (:classes imported-block)))))
 
 (deftest import-block-in-different-graph
-  (let [conn (db-test/create-conn-with-blocks
-              {:properties {:num-many {:logseq.property/type :number
-                                       :db/cardinality :many
-                                       :block/title "Num Many"
-                                       :logseq.property/hide? true}}
-               :classes {:MyClass {:block/title "My Class"
-                                   :build/class-properties [:default-many :p1]}}
-               :pages-and-blocks
-               [{:page {:block/title "page1"}
-                 :blocks [{:block/title "export"
-                           :build/properties {:num-many #{3 6 9}}
-                           :build/tags [:MyClass]}]}]})
+  (let [original-data
+        {:properties {:user.property/num-many
+                      {:logseq.property/type :number
+                       :db/cardinality :db.cardinality/many
+                       :block/title "Num Many"
+                       :logseq.property/hide? true}
+                      :user.property/p1
+                      {:db/cardinality :db.cardinality/one,
+                       :logseq.property/type :default,
+                       :block/title "p1"}}
+         :classes {:user.class/MyClass
+                   {:block/title "My Class"
+                    :build/class-properties [:user.property/num-many :user.property/p1]}}
+         :pages-and-blocks
+         [{:page {:block/title "page1"}
+           :blocks [{:block/title "export"
+                     :build/properties {:user.property/num-many #{3 6 9}}
+                     :build/tags [:user.class/MyClass]}]}]}
+        conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn-with-blocks
                {:pages-and-blocks [{:page {:block/title "page2"}
                                     :blocks [{:block/title "import"}
                                              {:block/title "import2"}]}]})
-        export-block (db-test/find-block-by-content @conn "export")
-        import-block* (db-test/find-block-by-content @conn2 "import")
-        {:keys [init-tx block-props-tx] :as _txs}
-        (->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)])
-             (sqlite-export/build-import @conn2 {:current-block import-block*}))
-        _ (assert (nil? (d/entity @conn2 :user.property/num-many)) "Does not have imported property")
-        _ (d/transact! conn2 init-tx)
-        _ (d/transact! conn2 block-props-tx)
-        ;; _ (cljs.pprint/pprint _txs)
-        import-block (d/entity @conn2 (:db/id import-block*))]
+        imported-block (export-block-and-import-to-another-block conn conn2 "export" "import")]
 
-    (is (ldb/property? (d/entity @conn2 :user.property/num-many))
-        "New user property is imported")
-    (is (= "Num Many"
-           (:block/title (d/entity @conn2 :user.property/num-many))))
-    (is (= {:db/cardinality :db.cardinality/many, :logseq.property/type :number, :logseq.property/hide? true}
-           (db-property/get-property-schema (d/entity @conn2 :user.property/num-many)))
-        "Imported property has correct schema properties")
+    (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
+           (:build/block imported-block))
+        "Imported block equals exported block")
+    (is (= (:properties original-data) (:properties imported-block)))
+    (is (= (:classes original-data) (:classes imported-block)))
 
-    (is (= "My Class"
-           (:block/title (d/entity @conn2 :user.class/MyClass))))
-    (is (= {:logseq.property.class/properties #{"default-many" "p1"}
-            :block/tags [:logseq.class/Tag]
-            :logseq.property/parent :logseq.class/Root}
-           (db-test/readable-properties (d/entity @conn2 :user.class/MyClass)))
-        "New user class has correct tag and properties")
-    (is (ldb/property? (d/entity @conn2 :user.property/p1))
-        "New class property is property")
+    (testing "same import in another block"
+      (let [imported-block (export-block-and-import-to-another-block conn conn2 "export" "import2")]
+        (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
+               (:build/block imported-block))
+            "Imported block equals exported block")
+        (is (= (:properties original-data) (:properties imported-block)))
+        (is (= (:classes original-data) (:classes imported-block)))))))
 
-    (is (= "export" (:block/title import-block))
-        "imported block title equals exported one")
-    (is (= {:user.property/num-many #{3 6 9}
-            :block/tags [:user.class/MyClass]}
-           (db-test/readable-properties import-block))
-        "imported block properties equals exported one")
+(deftest import-block-with-block-ref
+  (let [page-uuid (random-uuid)
+        original-data
+        {:pages-and-blocks
+         [{:page {:block/title "page1"}
+           :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}]}
+          {:page {:block/title "another page" :block/uuid page-uuid}}]}
+        conn (db-test/create-conn-with-blocks original-data)
+        conn2 (db-test/create-conn-with-blocks
+               {:pages-and-blocks [{:page {:block/title "page2"}
+                                    :blocks [{:block/title "import"}]}]})
+        imported-block (export-block-and-import-to-another-block conn conn2 #"page ref" "import")]
 
-    (testing "importing a 2nd time is idempotent"
-      (let [import-block2* (db-test/find-block-by-content @conn2 "import2")
-            {:keys [init-tx block-props-tx] :as _txs}
-            (->> (sqlite-export/build-block-export @conn [:block/uuid (:block/uuid export-block)])
-                 (sqlite-export/build-import @conn2 {:current-block import-block2*}))
-            _ (assert (empty? block-props-tx) "This is empty for properties that already exist and thus no transacted")
-            _ (d/transact! conn2 init-tx)
-            import-block2 (d/entity @conn2 (:db/id import-block2*))]
-        (is (= "export" (:block/title import-block2))
-            "imported block title equals exported one")
-        (is (= {:user.property/num-many #{3 6 9}
-                :block/tags [:user.class/MyClass]}
-               (db-test/readable-properties import-block))
-            "imported block properties equals exported one")))))
+    (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
+           (:build/block imported-block))
+        "Imported block equals exported block")
+    (is (= (second (:pages-and-blocks original-data))
+           (first (:pages-and-blocks imported-block)))
+        "Imported page equals exported page of page ref")))
 
 (defn- export-page-and-import-to-another-graph
   "Exports given page from one graph/conn, imports it to a 2nd graph and then
@@ -165,7 +161,7 @@
 
     (import-second-time-appends-blocks conn conn2 "page1" original-data)))
 
-(deftest ^:focus import-page-with-different-ref-types
+(deftest import-page-with-different-ref-types
   (let [block-uuid (random-uuid)
         class-uuid (random-uuid)
         page-uuid (random-uuid)
@@ -196,7 +192,6 @@
         "Page's properties are imported")
     (is (= (:classes original-data) (:classes full-imported-page))
         "Page's classes are imported")
-    ;; (cljs.pprint/pprint (:pages-and-blocks full-imported-page))
     (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page))
         "Page's blocks are imported")