add graph export support for property + class refs

Also fixed class + property refs forgetting keep-uuid?.
Also moved test helpers

deps/db/src/logseq/db/sqlite/build.cljs

@@ -598,7 +598,7 @@
     ;; (when (seq new-classes) (prn :new-classes new-classes))
     {:classes classes' :properties properties'}))
 
-(defn get-possible-referenced-uuids
+(defn- get-possible-referenced-uuids
   "Gets all possible ref uuids from either [:block/uuid X] or {:build/journal X}. Uuid scraping
    is aggressive so some uuids may not be referenced"
   [input-map]

deps/db/src/logseq/db/sqlite/export.cljs

@@ -94,7 +94,7 @@
                                             (-> (disj db-property/schema-properties :logseq.property/classes)
                                                 (conj :block/title)))
                          include-uuid?
-                         (assoc :block/uuid (:block/uuid property))
+                         (assoc :block/uuid (:block/uuid property) :build/keep-uuid? true)
                          include-timestamps?
                          (merge (select-keys property [:block/created-at :block/updated-at]))
                          (and (not shallow-copy?) (:logseq.property/classes property))
@@ -129,7 +129,7 @@
               :or {include-parents? true}}]
   (cond-> (select-keys class-ent [:block/title])
     include-uuid?
-    (assoc :block/uuid (:block/uuid class-ent))
+    (assoc :block/uuid (:block/uuid class-ent) :build/keep-uuid? true)
     include-timestamps?
     (merge (select-keys class-ent [:block/created-at :block/updated-at]))
     (and (:logseq.property.class/properties class-ent) (not shallow-copy?))
@@ -496,23 +496,12 @@
   [db options]
   (let [page-ids (concat (map :e (d/datoms db :avet :block/tags :logseq.class/Page))
                          (map :e (d/datoms db :avet :block/tags :logseq.class/Journal)))
-        content-ref-uuids (get-graph-content-ref-uuids db)
         page-exports (mapv (fn [eid]
                              (let [page-blocks* (get-page-blocks db eid)]
                                (build-page-export* db eid page-blocks* (merge options {:include-uuid-fn (constantly true)}))))
                            page-ids)
-        all-ref-uuids (set/union content-ref-uuids (set (mapcat :pvalue-uuids page-exports)))
         pages-export (apply merge-export-maps page-exports)
-        ;; Only way to ensure all pvalue-uuids present is to remove all non-ref uuids after
-        remove-uuid-if-not-ref (fn [m] (if (contains? all-ref-uuids (:block/uuid m))
-                                         m
-                                         (dissoc m :block/uuid :build/keep-uuid? true)))
-        pages-export' (update pages-export :pages-and-blocks
-                              (fn [pages-and-blocks]
-                                (mapv (fn [{:keys [page blocks]}]
-                                        {:page (remove-uuid-if-not-ref page)
-                                         :blocks (sqlite-build/update-each-block blocks remove-uuid-if-not-ref)})
-                                      pages-and-blocks)))]
+        pages-export' (assoc pages-export :pvalue-uuids (set (mapcat :pvalue-uuids page-exports)))]
     pages-export'))
 
 (defn- build-graph-files
@@ -523,16 +512,38 @@
                 (select-keys % [:file/path :file/content :file/created-at :file/last-modified-at])
                 (select-keys % [:file/path :file/content])))))
 
+(defn remove-uuids-if-not-ref [export-map all-ref-uuids]
+  (let [remove-uuid-if-not-ref (fn [m] (if (contains? all-ref-uuids (:block/uuid m))
+                                         m
+                                         (dissoc m :block/uuid :build/keep-uuid?)))]
+    (-> export-map
+        (update :classes update-vals remove-uuid-if-not-ref)
+        (update :properties update-vals remove-uuid-if-not-ref)
+        (update :pages-and-blocks
+                (fn [pages-and-blocks]
+                  (mapv (fn [{:keys [page blocks]}]
+                          {:page (remove-uuid-if-not-ref page)
+                           :blocks (sqlite-build/update-each-block blocks remove-uuid-if-not-ref)})
+                        pages-and-blocks))))))
+
 (defn- build-graph-export
   "Exports whole graph. Has the following options:
    * :include-timestamps? - When set timestamps are included on all blocks"
   [db options]
-  (let [ontology-export (build-graph-ontology-export db (select-keys options [:include-timestamps?]))
-        pages-export (build-graph-pages-export db (select-keys options [:include-timestamps?]))
+  (let [content-ref-uuids (get-graph-content-ref-uuids db)
+        ontology-options (merge (select-keys options [:include-timestamps?]) {:include-uuid? true})
+        ontology-export (build-graph-ontology-export db ontology-options)
+        ontology-pvalue-uuids (set (concat (mapcat get-pvalue-uuids (vals (:properties ontology-export)))
+                                           (mapcat get-pvalue-uuids (vals (:classes ontology-export)))))
+        pages-export (build-graph-pages-export db options)
+        all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export))
         files (build-graph-files db options)
-        graph-export (merge (merge-export-maps ontology-export pages-export)
-                            {::graph-files files})]
-    graph-export))
+        graph-export (merge-export-maps ontology-export pages-export)
+        ;; Remove all non-ref uuids after all nodes are built.
+        ;; Only way to ensure all pvalue uuids present across block types
+        graph-export' (remove-uuids-if-not-ref graph-export all-ref-uuids)]
+    (merge graph-export'
+           {::graph-files files})))
 
 (defn- find-undefined-classes-and-properties [{:keys [classes properties pages-and-blocks]}]
   (let [referenced-classes

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -39,6 +39,54 @@
     (sqlite-export/build-export @import-conn {:export-type :block
                                               :block-id (:db/id import-block)})))
 
+(defn- export-page-and-import-to-another-graph
+  "Exports given page from one graph/conn, imports it to a 2nd graph, validates
+  it and then exports the page from the 2nd graph"
+  [export-conn import-conn page-title]
+  (let [page (db-test/find-page-by-title @export-conn page-title)
+        {:keys [init-tx block-props-tx] :as _txs}
+        (-> (sqlite-export/build-export @export-conn {:export-type :page :page-id (:db/id page)})
+            ;; ((fn [x] (cljs.pprint/pprint {:export x}) x))
+            (sqlite-export/build-import @import-conn {}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)
+        _ (validate-db @import-conn)
+        page2 (db-test/find-page-by-title @import-conn page-title)]
+    (sqlite-export/build-export @import-conn {:export-type :page :page-id (:db/id page2)})))
+
+(defn- import-second-time-assertions [conn conn2 page-title original-data
+                                      & {:keys [transform-expected-blocks]
+                                         :or {transform-expected-blocks (fn [bs] (into bs bs))}}]
+  (let [page (db-test/find-page-by-title @conn2 page-title)
+        imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
+        updated-page (db-test/find-page-by-title @conn2 page-title)
+        expected-page-and-blocks
+        (update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks)]
+
+    ;; Assume first page is one being imported for now
+    (is (= (first expected-page-and-blocks)
+           (first (:pages-and-blocks imported-page)))
+        "Blocks are appended to existing page")
+    (is (= (:block/created-at page) (:block/created-at updated-page))
+        "Existing page didn't get re-created")
+    (is (= (:block/updated-at page) (:block/updated-at updated-page))
+        "Existing page didn't get updated")))
+
+(defn- export-graph-and-import-to-another-graph
+  "Exports graph and imports it to a 2nd graph, validates it and then exports the 2nd graph"
+  [export-conn import-conn export-options]
+  (let [{:keys [init-tx block-props-tx misc-tx] :as _txs}
+        (-> (sqlite-export/build-export @export-conn {:export-type :graph :graph-options export-options})
+            (sqlite-export/build-import @import-conn {}))
+        ;; _ (cljs.pprint/pprint _txs)
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)
+        _ (d/transact! import-conn misc-tx)
+        _ (validate-db @import-conn)
+        imported-graph (sqlite-export/build-export @import-conn {:export-type :graph :graph-options export-options})]
+    imported-graph))
+
 (defn- expand-properties
   "Add default values to properties of an input export map to test against a
   db-based export map"
@@ -177,40 +225,6 @@
            (first (:pages-and-blocks imported-block)))
         "Imported page equals exported page of page ref")))
 
-(defn- export-page-and-import-to-another-graph
-  "Exports given page from one graph/conn, imports it to a 2nd graph, validates
-  it and then exports the page from the 2nd graph"
-  [export-conn import-conn page-title]
-  (let [page (db-test/find-page-by-title @export-conn page-title)
-        {:keys [init-tx block-props-tx] :as _txs}
-        (-> (sqlite-export/build-export @export-conn {:export-type :page :page-id (:db/id page)})
-            ;; ((fn [x] (cljs.pprint/pprint {:export x}) x))
-            (sqlite-export/build-import @import-conn {}))
-        ;; _ (cljs.pprint/pprint _txs)
-        _ (d/transact! import-conn init-tx)
-        _ (d/transact! import-conn block-props-tx)
-        _ (validate-db @import-conn)
-        page2 (db-test/find-page-by-title @import-conn page-title)]
-    (sqlite-export/build-export @import-conn {:export-type :page :page-id (:db/id page2)})))
-
-(defn- import-second-time-assertions [conn conn2 page-title original-data
-                                      & {:keys [transform-expected-blocks]
-                                         :or {transform-expected-blocks (fn [bs] (into bs bs))}}]
-  (let [page (db-test/find-page-by-title @conn2 page-title)
-        imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
-        updated-page (db-test/find-page-by-title @conn2 page-title)
-        expected-page-and-blocks
-        (update-in (:pages-and-blocks original-data) [0 :blocks] transform-expected-blocks)]
-
-    ;; Assume first page is one being imported for now
-    (is (= (first expected-page-and-blocks)
-           (first (:pages-and-blocks imported-page)))
-        "Blocks are appended to existing page")
-    (is (= (:block/created-at page) (:block/created-at updated-page))
-        "Existing page didn't get re-created")
-    (is (= (:block/updated-at page) (:block/updated-at updated-page))
-        "Existing page didn't get updated")))
-
 ;; Tests a variety of blocks including block children with new properties, blocks with users classes
 ;; and blocks with built-in properties and classes
 (deftest import-page-with-different-blocks
@@ -517,10 +531,16 @@
   (let [internal-block-uuid (random-uuid)
         favorited-uuid (random-uuid)
         block-object-uuid (random-uuid)
+        block-object-uuid2 (random-uuid)
         closed-value-uuid (random-uuid)
+        property-uuid (random-uuid)
+        class-uuid (random-uuid)
         original-data
         {:properties
-         {:user.property/num {:logseq.property/type :number}
+         {:user.property/num {:logseq.property/type :number
+                              :block/uuid property-uuid
+                              :build/keep-uuid? true
+                              :build/properties {:user.property/node #{[:block/uuid block-object-uuid2]}}}
           :user.property/default-closed
           {:logseq.property/type :default
            :build/closed-values [{:value "joy" :uuid closed-value-uuid}
@@ -532,7 +552,9 @@
                                :db/cardinality :db.cardinality/many
                                :build/property-classes [:user.class/MyClass]}}
          :classes
-         {:user.class/MyClass {:build/properties {:user.property/url "https://example.com/MyClass"}}
+         {:user.class/MyClass {:build/properties {:user.property/url "https://example.com/MyClass"}
+                               :block/uuid class-uuid
+                               :build/keep-uuid? true}
           :user.class/MyClass2 {:build/class-parent :user.class/MyClass
                                 :build/properties {:logseq.property/description "tests child class"}}}
          :pages-and-blocks
@@ -547,11 +569,17 @@
                      :build/tags [:user.class/MyClass]
                      :block/uuid block-object-uuid
                      :build/keep-uuid? true}
+                    {:block/title "myclass object 2"
+                     :build/tags [:user.class/MyClass]
+                     :block/uuid block-object-uuid2
+                     :build/keep-uuid? true}
                     {:block/title "ref blocks"
                      :build/children
                      [{:block/title (str "internal block ref to " (page-ref/->page-ref internal-block-uuid))}
                       {:block/title "node block"
-                       :build/properties {:user.property/node #{[:block/uuid block-object-uuid]}}}]}]}
+                       :build/properties {:user.property/node #{[:block/uuid block-object-uuid]}}}
+                      {:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}
+                      {:block/title (str "class ref to " (page-ref/->page-ref class-uuid))}]}]}
           {:page {:build/journal 20250228 :build/properties {:user.property/num 1}}
            :blocks [{:block/title "journal block"}]}
           {:page {:build/journal 19650201}, :blocks []}
@@ -584,20 +612,6 @@
            :file/content "// comment"}]}]
     original-data))
 
-(defn- export-graph-and-import-to-another-graph
-  "Exports graph and imports it to a 2nd graph, validates it and then exports the 2nd graph"
-  [export-conn import-conn export-options]
-  (let [{:keys [init-tx block-props-tx misc-tx] :as _txs}
-        (-> (sqlite-export/build-export @export-conn {:export-type :graph :graph-options export-options})
-            (sqlite-export/build-import @import-conn {}))
-        ;; _ (cljs.pprint/pprint _txs)
-        _ (d/transact! import-conn init-tx)
-        _ (d/transact! import-conn block-props-tx)
-        _ (d/transact! import-conn misc-tx)
-        _ (validate-db @import-conn)
-        imported-graph (sqlite-export/build-export @import-conn {:export-type :graph :graph-options export-options})]
-    imported-graph))
-
 (deftest ^:focus import-graph
   (let [original-data (build-original-graph-data)
         conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
@@ -607,6 +621,8 @@
 
     ;; (cljs.pprint/pprint (set (:pages-and-blocks original-data)))
     ;; (cljs.pprint/pprint (set (:pages-and-blocks imported-graph)))
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (set (:pages-and-blocks original-data))
+    ;;                                                 (set (:pages-and-blocks imported-graph)))))
     (is (= (set (:pages-and-blocks original-data)) (set (:pages-and-blocks imported-graph))))
     (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
     (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))