enhance: introduce :build/keep-uuid? for export/import

to consistently keep uuids for all node types.
Allows page and block hacks to be removed. Marking blocks
this way was the right balance of efficient at preserving block relationships
while being easy to maintain

deps/db/src/logseq/db/sqlite/build.cljs

@@ -133,8 +133,8 @@
     (map second (re-seq page-ref/page-ref-re s))))
 
 (defn- ->block-tx [{:keys [build/properties] :as m} page-uuids all-idents page-id
-                   {properties-config :properties :keys [build-existing-tx? existing-page?]}]
-  (let [build-existing-tx?' (and build-existing-tx? (::existing-block? (meta m)) existing-page?)
+                   {properties-config :properties :keys [build-existing-tx?]}]
+  (let [build-existing-tx?' (and build-existing-tx? (::existing-block? (meta m)) (not (:build/keep-uuid? m)))
         block (if build-existing-tx?'
                 (select-keys m [:block/uuid])
                 {:db/id (new-db-id)
@@ -211,7 +211,7 @@
 (defn- build-properties-tx [properties page-uuids all-idents {:keys [build-existing-tx?]}]
   (let [properties' (if build-existing-tx?
                       (->> properties
-                           (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/new-property? v)))))
+                           (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/keep-uuid? v)))))
                            (into {}))
                       properties)
         property-db-ids (->> (keys properties')
@@ -225,7 +225,7 @@
 (defn- build-classes-tx [classes properties-config uuid-maps all-idents {:keys [build-existing-tx?]}]
   (let [classes' (if build-existing-tx?
                    (->> classes
-                        (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/new-class? v)))))
+                        (remove (fn [[_ v]] (and (:block/uuid v) (not (:build/keep-uuid? v)))))
                         (into {}))
                    classes)
         class-db-ids (->> (keys classes')
@@ -280,14 +280,16 @@
                         [:block/title :string]
                         [:build/children {:optional true} [:vector [:ref ::block]]]
                         [:build/properties {:optional true} User-properties]
-                        [:build/tags {:optional true} [:vector Class]]]}}
+                        [:build/tags {:optional true} [:vector Class]]
+                        [:build/keep-uuid? {:optional true} :boolean]]}}
    [:page [:and
            [:map
             [:block/uuid {:optional true} :uuid]
             [:block/title {:optional true} :string]
             [:build/journal {:optional true} :int]
             [:build/properties {:optional true} User-properties]
-            [:build/tags {:optional true} [:vector Class]]]
+            [:build/tags {:optional true} [:vector Class]]
+            [:build/keep-uuid? {:optional true} :boolean]]
            [:fn {:error/message ":block/title, :block/uuid or :build/journal required"
                  :error/path [:block/title]}
             (fn [m]
@@ -307,7 +309,8 @@
                [:value [:or :string :double]]
                [:uuid {:optional true} :uuid]
                [:icon {:optional true} :map]]]]
-    [:build/property-classes {:optional true} [:vector Class]]]])
+    [:build/property-classes {:optional true} [:vector Class]]
+    [:build/keep-uuid? {:optional true} :boolean]]])
 
 (def Classes
   [:map-of
@@ -315,7 +318,8 @@
    [:map
     [:build/properties {:optional true} User-properties]
     [:build/class-parent {:optional true} Class]
-    [:build/class-properties {:optional true} [:vector Property]]]])
+    [:build/class-properties {:optional true} [:vector Property]]
+    [:build/keep-uuid? {:optional true} :boolean]]])
 
 (def Options
   [:map
@@ -413,10 +417,7 @@
   (vec
    (mapcat
     (fn [{:keys [page blocks]}]
-      (let [;; For a page to be ignored it's important that it only sends a {:block/uuid UUID} map.
-            ;; This allows import processes to append blocks to an existing page or to create a page
-            ;; that is referenced by another page
-            ignore-page-tx? (and build-existing-tx? (not (::new-page? (meta page))) (= '(:block/uuid) (keys page)))
+      (let [ignore-page-tx? (and build-existing-tx? (not (::new-page? (meta page))) (not (:build/keep-uuid? page)))
             page' (if ignore-page-tx?
                     page
                     (merge
@@ -428,8 +429,7 @@
                      (dissoc page :db/id :block/name :block/title)))
             page-id-fn' (if (and build-existing-tx? (not (::new-page? (meta page))))
                           #(vector :block/uuid (:block/uuid %))
-                          page-id-fn)
-            opts' (assoc opts :existing-page? (and build-existing-tx? (not (::new-page? (meta page)))))]
+                          page-id-fn)]
         (into
          ;; page tx
          (if ignore-page-tx?
@@ -438,7 +438,7 @@
          ;; blocks tx
          (reduce (fn [acc m]
                    (into acc
-                         (->block-tx m page-uuids all-idents (page-id-fn' page') opts')))
+                         (->block-tx m page-uuids all-idents (page-id-fn' page') opts)))
                  []
                  blocks))))
     pages-and-blocks)))
@@ -641,11 +641,13 @@
        * :build/journal - Define a journal pages as an integer e.g. 20240101 is Jan 1, 2024. :block/title
          is not required if using this since it generates one
        * :build/properties - Defines properties on a page
+       * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
      * :blocks - This is a vec of datascript attribute maps for blocks with
        :block/title required. e.g. `{:block/title \"bar\"}`. Additional keys available:
        * :build/children - A vec of blocks that are nested (indented) under the current block.
           Allows for outlines to be expressed to whatever depth
        * :build/properties - Defines properties on a block
+       * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
    * :properties - This is a map to configure properties where the keys are property name keywords
      and the values are maps of datascript attributes e.g. `{:logseq.property/type :checkbox}`.
      Additional keys available:
@@ -654,18 +656,21 @@
      * :build/property-classes - Vec of class name keywords. Defines a property's range classes
      * :build/properties-ref-types - Map of internal ref types to public ref types that are valid only for this property.
        Useful when remapping value ref types e.g. for :logseq.property/default-value
+     * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
    * :classes - This is a map to configure classes where the keys are class name keywords
      and the values are maps of datascript attributes e.g. `{:block/title \"Foo\"}`.
      Additional keys available:
      * :build/properties - Define properties on a class page
      * :build/class-parent - Add a class parent by its keyword name
      * :build/class-properties - Vec of property name keywords. Defines properties that a class gives to its objects
+     * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
   * :graph-namespace - namespace to use for db-ident creation. Useful when importing an ontology
   * :auto-create-ontology? - When set to true, creates properties and classes from their use.
     See auto-create-ontology for more details
   * :build-existing-tx? - When set to true, blocks, pages, properties and classes with :block/uuid are treated as
      existing in DB and are skipped for creation. This is useful for building tx on existing DBs e.g. for importing.
-     Blocks are updated with any attributes passed to it while all other node types are ignored for update.
+     Blocks are updated with any attributes passed to it while all other node types are ignored for update unless
+     :build/keep-uuid? is set.
   * :page-id-fn - custom fn that returns ent lookup id for page refs e.g. `[:block/uuid X]`
     Default is :db/id
 

deps/db/src/logseq/db/sqlite/export.cljs

@@ -139,7 +139,7 @@
 
 (defn- build-entity-export
   "Given entity and optional existing properties, build an EDN export map"
-  [db entity {:keys [properties include-uuid-fn] :or {include-uuid-fn (constantly false)}}]
+  [db entity {:keys [properties include-uuid-fn keep-uuid?] :or {include-uuid-fn (constantly false)}}]
   (let [ent-properties (dissoc (db-property/properties entity) :block/tags)
         new-user-property-ids (->> (keys ent-properties)
                                    (concat (->> (:block/tags entity)
@@ -153,6 +153,8 @@
         build-block (cond-> {:block/title (block-title entity)}
                       (include-uuid-fn (:block/uuid entity))
                       (assoc :block/uuid (:block/uuid entity))
+                      keep-uuid?
+                      (assoc :build/keep-uuid? true)
                       (seq build-tags)
                       (assoc :build/tags build-tags)
                       (seq ent-properties)
@@ -197,19 +199,20 @@
         content-ref-pages (filter #(or (ldb/internal-page? %) (ldb/journal? %)) content-ref-ents)
         content-ref-properties (when-let [prop-ids (seq (map :db/ident (filter ldb/property? content-ref-ents)))]
                                  (update-vals (build-export-properties db prop-ids {:include-uuid? true})
-                                              #(merge % {:build/new-property? true})))
+                                              #(merge % {:build/keep-uuid? true})))
         content-ref-classes (when-let [class-ents (seq (filter ldb/class? content-ref-ents))]
                               (->> class-ents
                                    ;; TODO: Export class parents when there's ability to control granularity of export
                                    (map #(vector (:db/ident %)
                                                  (assoc (build-export-class % {:include-parents? false :include-uuid? true})
-                                                        :build/new-class? true)))
+                                                        :build/keep-uuid? true)))
                                    (into {})))]
     {:content-ref-uuids content-ref-uuids
      :content-ref-ents content-ref-ents
      :properties content-ref-properties
      :classes content-ref-classes
-     :pages-and-blocks (mapv #(hash-map :page (assoc (shallow-copy-page %) :block/uuid (:block/uuid %)))
+     :pages-and-blocks (mapv #(hash-map :page (merge (shallow-copy-page %)
+                                                     {:block/uuid (:block/uuid %) :build/keep-uuid? true}))
                              content-ref-pages)}))
 
 (defn build-block-export
@@ -228,7 +231,7 @@
 (defn- build-blocks-tree
   "Given a page's block entities, returns the blocks in a sqlite.build EDN format
    and all properties and classes used in these blocks"
-  [db blocks {:keys [include-uuid-fn]}]
+  [db blocks opts]
   (let [*properties (atom {})
         *classes (atom {})
         *pvalue-uuids (atom #{})
@@ -237,7 +240,7 @@
         build-block (fn build-block [block*]
                       (let [child-nodes (mapv build-block (get children (:db/id block*) []))
                             {:build/keys [block] :keys [properties classes]}
-                            (build-entity-export db block* {:properties @*properties :include-uuid-fn include-uuid-fn})
+                            (build-entity-export db block* (assoc opts :properties @*properties))
                             new-pvalue-uuids (get-pvalue-uuids block)]
                         (when (seq properties) (swap! *properties merge properties))
                         (when (seq classes) (swap! *classes merge classes))
@@ -263,7 +266,9 @@
                ((fn [m] (dissoc m page-entity)))
                (map (fn [[parent-page-ent blocks]]
                       ;; Don't export pvalue-uuids of uuid blocks to keep export shallower
-                      (merge (build-blocks-tree db (sort-by :block/order blocks) {:include-uuid-fn (constantly true)})
+                      (merge (build-blocks-tree db
+                                                (sort-by :block/order blocks)
+                                                {:include-uuid-fn (constantly true) :keep-uuid? true})
                              {:page (shallow-copy-page parent-page-ent)})))))]
     {:properties (apply merge (map :properties uuid-block-pages))
      :classes (apply merge (map :classes uuid-block-pages))

deps/db/test/logseq/db/sqlite/build_test.cljs

@@ -127,8 +127,8 @@
         page-uuid (random-uuid)
         property-uuid (random-uuid)
         conn (db-test/create-conn-with-blocks
-              {:classes {:C1 {:block/uuid class-uuid :build/new-class? true}}
-               :properties {:p1 {:block/uuid property-uuid :build/new-property? true}}
+              {:classes {:C1 {:block/uuid class-uuid :build/keep-uuid? true}}
+               :properties {:p1 {:block/uuid property-uuid :build/keep-uuid? true}}
                :build-existing-tx? true
                :pages-and-blocks
                [{:page {:block/title "page 1"}
@@ -138,8 +138,8 @@
                           {:block/title (str "class ref to " (page-ref/->page-ref class-uuid))}
                           {:block/title (str "inline class ref to #" (page-ref/->page-ref class-uuid))}
                           {:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}
-                          {:block/title "hi" :block/uuid block-uuid}]}
-                {:page {:block/title "another page" :block/uuid page-uuid}}]})
+                          {:block/title "hi" :block/uuid block-uuid :build/keep-uuid? true}]}
+                {:page {:block/title "another page" :block/uuid page-uuid :build/keep-uuid? true}}]})
         block-with-named-page-ref (db-test/find-block-by-content @conn #"^named page ref")
         block-with-page-ref (db-test/find-block-by-content @conn #"^page ref")
         block-with-class-ref (db-test/find-block-by-content @conn #"^class ref")

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -86,7 +86,7 @@
         {:pages-and-blocks
          [{:page {:block/title "page1"}
            :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}]}
-          {:page {:block/title "another page" :block/uuid page-uuid}}]}
+          {:page {:block/title "another page" :block/uuid page-uuid :build/keep-uuid? true}}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn-with-blocks
                {:pages-and-blocks [{:page {:block/title "page2"}
@@ -168,10 +168,10 @@
         property-uuid (random-uuid)
         journal-uuid (random-uuid)
         original-data
-        {:classes {:user.class/C1 {:block/title "C1" :block/uuid class-uuid :build/new-class? true}}
+        {:classes {:user.class/C1 {:block/title "C1" :block/uuid class-uuid :build/keep-uuid? true}}
          :properties {:user.property/p1
                       {:db/cardinality :db.cardinality/one, :logseq.property/type :default
-                       :block/uuid property-uuid :block/title "p1" :build/new-property? true}}
+                       :block/uuid property-uuid :block/title "p1" :build/keep-uuid? true}}
          :pages-and-blocks
          [{:page {:block/title "page1"}
            :blocks [{:block/title (str "page ref to " (page-ref/->page-ref page-uuid))}
@@ -181,9 +181,9 @@
                     {:block/title (str "property ref to " (page-ref/->page-ref property-uuid))}
                     {:block/title (str "journal ref to " (page-ref/->page-ref journal-uuid))}]}
           {:page {:block/title "page with block ref"}
-           :blocks [{:block/title "hi" :block/uuid block-uuid}]}
-          {:page {:block/title "another page" :block/uuid page-uuid}}
-          {:page {:build/journal 20250207 :block/uuid journal-uuid}}]}
+           :blocks [{:block/title "hi" :block/uuid block-uuid :build/keep-uuid? true}]}
+          {:page {:block/title "another page" :block/uuid page-uuid :build/keep-uuid? true}}
+          {:page {:build/journal 20250207 :block/uuid journal-uuid :build/keep-uuid? true}}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn)
         full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
@@ -253,7 +253,8 @@
           {:page {:block/title "Blocks"}
            :blocks [{:block/title "myclass object"
                      :build/tags [:user.class/MyClass]
-                     :block/uuid block-object-uuid}]}]}
+                     :block/uuid block-object-uuid
+                     :build/keep-uuid? true}]}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn)
         full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]