refactor: use namespaced keyword for block export

instead of confusing :build/block. Also improved related tests,
simplified import steps, added some missing sqlite.build docs,
and fixed :build/uuid not working for some existing journals.

deps/db/script/create_graph.cljs

@@ -28,9 +28,10 @@
         sqlite-build-edn (merge {:auto-create-ontology? true}
                                 (-> (resolve-path edn-path) fs/readFileSync str edn/read-string))
         conn (outliner-cli/init-conn dir db-name {:classpath (cp/get-classpath) :import-type :cli/create-graph})
-        {:keys [init-tx block-props-tx]} (outliner-cli/build-blocks-tx sqlite-build-edn)]
+        {:keys [init-tx block-props-tx] :as _txs} (outliner-cli/build-blocks-tx sqlite-build-edn)]
     (println "Generating" (count (filter :block/name init-tx)) "pages and"
              (count (filter :block/title init-tx)) "blocks ...")
+    ;; (cljs.pprint/pprint _txs)
     (d/transact! conn init-tx)
     (d/transact! conn block-props-tx)
     (println "Created graph" (str db-name "!"))))

deps/db/src/logseq/db/sqlite/build.cljs

@@ -544,7 +544,7 @@
                                                    :block/uuid
                                                    (or (:block/uuid page) (common-uuid/gen-uuid :journal-page-uuid date-int))
                                                    :block/tags :logseq.class/Journal})
-                                           (with-meta {::new-page? true})))))
+                                           (with-meta {::new-page? (not (:block/uuid page))})))))
                            m))]
     ;; Order matters as some steps depend on previous step having prepared blocks or pages in a certain way
     (->> pages-and-blocks
@@ -641,12 +641,14 @@
        * :build/journal - Define a journal pages as an integer e.g. 20240101 is Jan 1, 2024. :block/title
          is not required if using this since it generates one
        * :build/properties - Defines properties on a page
+       * :build/tags - Defines tags on a page
        * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
      * :blocks - This is a vec of datascript attribute maps for blocks with
        :block/title required. e.g. `{:block/title \"bar\"}`. Additional keys available:
        * :build/children - A vec of blocks that are nested (indented) under the current block.
           Allows for outlines to be expressed to whatever depth
        * :build/properties - Defines properties on a block
+       * :build/tags - Defines tags on a block
        * :build/keep-uuid? - Keeps :block/uuid because another block depends on it
    * :properties - This is a map to configure properties where the keys are property name keywords
      and the values are maps of datascript attributes e.g. `{:logseq.property/type :checkbox}`.

deps/db/src/logseq/db/sqlite/export.cljs

@@ -137,8 +137,9 @@
          (map #(vector (:db/ident %) (build-export-class % {:include-parents? false})))
          (into {}))))
 
-(defn- build-entity-export
-  "Given entity and optional existing properties, build an EDN export map"
+(defn- build-node-export
+  "Given a block/page entity and optional existing properties, build an export map of its
+   tags and properties"
   [db entity {:keys [properties include-uuid-fn keep-uuid?] :or {include-uuid-fn (constantly false)}}]
   (let [ent-properties (dissoc (db-property/properties entity) :block/tags)
         new-user-property-ids (->> (keys ent-properties)
@@ -150,7 +151,7 @@
                                    (remove #(get properties %)))
         new-properties (build-export-properties db new-user-property-ids {})
         build-tags (when (seq (:block/tags entity)) (->build-tags (:block/tags entity)))
-        build-block (cond-> {:block/title (block-title entity)}
+        build-node (cond-> {:block/title (block-title entity)}
                       (include-uuid-fn (:block/uuid entity))
                       (assoc :block/uuid (:block/uuid entity))
                       keep-uuid?
@@ -160,8 +161,8 @@
                       (seq ent-properties)
                       (assoc :build/properties
                              (buildable-properties db ent-properties (merge properties new-properties))))
-        new-classes (build-export-block-classes db build-block (:block/tags entity))]
-    (cond-> {:build/block build-block}
+        new-classes (build-export-block-classes db build-node (:block/tags entity))]
+    (cond-> {:node build-node}
       (seq new-classes)
       (assoc :classes new-classes)
       (seq new-properties)
@@ -219,10 +220,10 @@
   [db eid]
   (let [block-entity (d/entity db eid)
         {:keys [content-ref-uuids _content-ref-ents] :as content-ref-export} (build-content-ref-export db [block-entity])
-        block-export* (build-entity-export db block-entity {:include-uuid-fn content-ref-uuids})
-        pvalue-uuids (get-pvalue-uuids (:build/block block-export*))
-        block-export (assoc (merge-export-maps block-export* content-ref-export)
-                            :build/block (:build/block block-export*))]
+        node-export (build-node-export db block-entity {:include-uuid-fn content-ref-uuids})
+        pvalue-uuids (get-pvalue-uuids (:node node-export))
+        block-export (merge {::block (:node node-export)}
+                            (merge-export-maps node-export content-ref-export))]
     ;; Maybe add support for this later
     (when (seq pvalue-uuids)
       (throw (ex-info "Exporting a block with :node block objects is not supported" {})))
@@ -239,13 +240,13 @@
         children (group-by #(get-in % [:block/parent :db/id]) blocks)
         build-block (fn build-block [block*]
                       (let [child-nodes (mapv build-block (get children (:db/id block*) []))
-                            {:build/keys [block] :keys [properties classes]}
-                            (build-entity-export db block* (assoc opts :properties @*properties))
-                            new-pvalue-uuids (get-pvalue-uuids block)]
+                            {:keys [node properties classes]}
+                            (build-node-export db block* (assoc opts :properties @*properties))
+                            new-pvalue-uuids (get-pvalue-uuids node)]
                         (when (seq properties) (swap! *properties merge properties))
                         (when (seq classes) (swap! *classes merge classes))
                         (when (seq new-pvalue-uuids) (swap! *pvalue-uuids into new-pvalue-uuids))
-                        (cond-> block
+                        (cond-> node
                           (seq child-nodes) (assoc :build/children child-nodes))))
         roots (remove #(contains? id-map (get-in % [:block/parent :db/id])) blocks)
         exported-blocks (mapv build-block roots)]
@@ -288,8 +289,8 @@
         {:keys [blocks properties classes pvalue-uuids]}
         (build-blocks-tree db page-blocks {:include-uuid-fn content-ref-uuids})
         uuid-block-export (build-uuid-block-export db pvalue-uuids content-ref-ents page-entity)
-        page-ent-export (build-entity-export db page-entity {:properties properties})
-        page (merge (dissoc (:build/block page-ent-export) :block/title)
+        page-ent-export (build-node-export db page-entity {:properties properties})
+        page (merge (dissoc (:node page-ent-export) :block/title)
                     (shallow-copy-page page-entity))
         page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}]
                             :properties properties
@@ -327,11 +328,25 @@
 
 ;; Import fns
 ;; ==========
-(defn- ->sqlite-build-options
+(defn- check-for-existing-entities
+  "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import.
+   Also checks for property conflicts between existing properties and properties to be imported"
   [db {:keys [pages-and-blocks classes properties]} property-conflicts]
   (cond-> {:build-existing-tx? true}
     (seq pages-and-blocks)
-    (assoc :pages-and-blocks pages-and-blocks)
+    (assoc :pages-and-blocks
+           (mapv (fn [m]
+                   (if-let [ent (some->> (get-in m [:page :build/journal])
+                                         (d/datoms db :avet :block/journal-day)
+                                         first
+                                         :e
+                                         (d/entity db))]
+                     (assoc-in m [:page :block/uuid] (:block/uuid ent))
+                     ;; For now only check page uniqueness by title. Could handle more uniqueness checks later
+                     (if-let [ent (some->> (get-in m [:page :block/title]) (ldb/get-case-page db))]
+                       (assoc-in m [:page :block/uuid] (:block/uuid ent))
+                       m)))
+                 pages-and-blocks))
     (seq classes)
     (assoc :classes
            (->> classes
@@ -359,7 +374,7 @@
 (defn- build-block-import-options
   "Builds options for sqlite-build to import into current-block"
   [current-block export-map]
-  (let [block (merge (:build/block export-map)
+  (let [block (merge (::block export-map)
                      {:block/uuid (:block/uuid current-block)
                       :block/page (select-keys (:block/page current-block) [:block/uuid])})
         pages-and-blocks
@@ -367,35 +382,17 @@
           :blocks [(dissoc block :block/page)]}]]
     (merge-export-maps export-map {:pages-and-blocks pages-and-blocks})))
 
-(defn- build-page-import-options
-  [db export-map]
-  (assert (map? (get-in export-map [:pages-and-blocks 0 :page])) "page export exists")
-  (if-let [ent (some->> (get-in export-map [:pages-and-blocks 0 :page :build/journal])
-                        (d/datoms db :avet :block/journal-day)
-                        first
-                        :e
-                        (d/entity db))]
-    (assoc-in export-map [:pages-and-blocks 0 :page] (select-keys ent [:block/uuid]))
-    ;; FIXME: Find an existing page more reliably than :block/title, :block/uuid?
-    (if-let [ent (some->> (get-in export-map [:pages-and-blocks 0 :page :block/title])
-                          (ldb/get-case-page db))]
-      (assoc-in export-map [:pages-and-blocks 0 :page] (select-keys ent [:block/uuid]))
-      export-map)))
-
 (defn build-import
   "Given an entity's export map, build the import tx to create it"
-  [db {:keys [current-block]} export-map*]
-  (let [export-map (cond current-block
-                         (build-block-import-options current-block export-map*)
-                         (:pages-and-blocks export-map*)
-                         (build-page-import-options db export-map*)
-                         :else
-                         export-map*)
+  [export-map* db {:keys [current-block]}]
+  (let [export-map (if (and (::block export-map*) current-block)
+                     (build-block-import-options current-block export-map*)
+                     export-map*)
         property-conflicts (atom [])
-        opts (->sqlite-build-options db export-map property-conflicts)]
+        export-map' (check-for-existing-entities db export-map property-conflicts)]
     (if (seq @property-conflicts)
       (do
         (js/console.error :property-conflicts @property-conflicts)
         {:error (str "The following imported properties conflict with the current graph: "
                      (pr-str (mapv :property-id @property-conflicts)))})
-      (sqlite-build/build-blocks-tx opts))))
+      (sqlite-build/build-blocks-tx export-map'))))

deps/db/src/logseq/db/test/helper.cljs

@@ -43,6 +43,18 @@
        first
        (d/entity db)))
 
+(defn find-journal-by-journal-day
+  [db journal-day]
+  (->> journal-day
+       (d/q
+        '[:find [?page ...]
+          :in $ ?journal-day
+          :where
+          [?page :block/journal-day ?journal-day]]
+        db)
+       first
+       (d/entity db)))
+
 (defn readable-properties
   "Returns an entity's properties and tags in readable form for assertions.
    tags are included here since they behave like properties on an ent"

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -3,7 +3,8 @@
             [datascript.core :as d]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db.sqlite.export :as sqlite-export]
-            [logseq.db.test.helper :as db-test]))
+            [logseq.db.test.helper :as db-test]
+            [logseq.common.util.date-time :as date-time-util]))
 
 (defn- export-block-and-import-to-another-block
   "Exports given block from one graph/conn, imports it to a 2nd block and then
@@ -12,8 +13,8 @@
   (let [export-block (db-test/find-block-by-content @export-conn export-block-content)
         import-block (db-test/find-block-by-content @import-conn import-block-content)
         {:keys [init-tx block-props-tx] :as _txs}
-        (->> (sqlite-export/build-block-export @export-conn [:block/uuid (:block/uuid export-block)])
-             (sqlite-export/build-import @import-conn {:current-block import-block}))
+        (-> (sqlite-export/build-block-export @export-conn [:block/uuid (:block/uuid export-block)])
+            (sqlite-export/build-import @import-conn {:current-block import-block}))
         ;; _ (cljs.pprint/pprint _txs)
         _ (d/transact! import-conn init-tx)
         _ (d/transact! import-conn block-props-tx)]
@@ -35,7 +36,7 @@
         imported-block (export-block-and-import-to-another-block conn conn "export" "import")]
 
     (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
-           (:build/block imported-block))
+           (::sqlite-export/block imported-block))
         "Imported block equals exported block")
     (is (= (:properties original-data) (:properties imported-block)))
     (is (= (:classes original-data) (:classes imported-block)))))
@@ -67,7 +68,7 @@
         imported-block (export-block-and-import-to-another-block conn conn2 "export" "import")]
 
     (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
-           (:build/block imported-block))
+           (::sqlite-export/block imported-block))
         "Imported block equals exported block")
     (is (= (:properties original-data) (:properties imported-block)))
     (is (= (:classes original-data) (:classes imported-block)))
@@ -75,7 +76,7 @@
     (testing "same import in another block"
       (let [imported-block (export-block-and-import-to-another-block conn conn2 "export" "import2")]
         (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
-               (:build/block imported-block))
+               (::sqlite-export/block imported-block))
             "Imported block equals exported block")
         (is (= (:properties original-data) (:properties imported-block)))
         (is (= (:classes original-data) (:classes imported-block)))))))
@@ -94,7 +95,7 @@
         imported-block (export-block-and-import-to-another-block conn conn2 #"page ref" "import")]
 
     (is (= (get-in original-data [:pages-and-blocks 0 :blocks 0])
-           (:build/block imported-block))
+           (::sqlite-export/block imported-block))
         "Imported block equals exported block")
     (is (= (second (:pages-and-blocks original-data))
            (first (:pages-and-blocks imported-block)))
@@ -106,21 +107,29 @@
   [export-conn import-conn page-title]
   (let [page (db-test/find-page-by-title @export-conn page-title)
         {:keys [init-tx block-props-tx] :as _txs}
-        (->> (sqlite-export/build-page-export @export-conn (:db/id page))
-            ;;  ((fn [x] (cljs.pprint/pprint {:export x}) x))
-             (sqlite-export/build-import @import-conn {}))
+        (-> (sqlite-export/build-page-export @export-conn (:db/id page))
+            ;; ((fn [x] (cljs.pprint/pprint {:export x}) x))
+            (sqlite-export/build-import @import-conn {}))
         ;; _ (cljs.pprint/pprint _txs)
         _ (d/transact! import-conn init-tx)
         _ (d/transact! import-conn block-props-tx)
         page2 (db-test/find-page-by-title @import-conn page-title)]
     (sqlite-export/build-page-export @import-conn (:db/id page2))))
 
-(defn- import-second-time-appends-blocks [conn conn2 page-title original-data]
-  (let [full-imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
+(defn- import-second-time-assertions [conn conn2 page-title original-data]
+  (let [page (db-test/find-page-by-title @conn2 page-title)
+        imported-page (export-page-and-import-to-another-graph conn conn2 page-title)
+        updated-page (db-test/find-page-by-title @conn2 page-title)
         expected-page-and-blocks
         (update-in (:pages-and-blocks original-data) [0 :blocks]
                    (fn [blocks] (into blocks blocks)))]
-    (is (= expected-page-and-blocks (:pages-and-blocks full-imported-page)))))
+
+    (is (= expected-page-and-blocks (:pages-and-blocks imported-page))
+        "Blocks are appended to existing page blocks")
+    (is (= (:block/created-at page) (:block/created-at updated-page))
+        "Existing page didn't get re-created")
+    (is (= (:block/updated-at page) (:block/updated-at updated-page))
+        "Existing page didn't get updated")))
 
 ;; Tests a variety of blocks including block children with new properties, blocks with users classes
 ;; and blocks with built-in properties and classes
@@ -150,16 +159,16 @@
                      :build/tags [:logseq.class/Task]}]}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn)
-        full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
+        imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
 
-    (is (= (:properties original-data) (:properties full-imported-page))
+    (is (= (:properties original-data) (:properties imported-page))
         "Page's properties are imported")
-    (is (= (:classes original-data) (:classes full-imported-page))
+    (is (= (:classes original-data) (:classes imported-page))
         "Page's classes are imported")
-    (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page))
+    (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page))
         "Page's blocks are imported")
 
-    (import-second-time-appends-blocks conn conn2 "page1" original-data)))
+    (import-second-time-assertions conn conn2 "page1" original-data)))
 
 (deftest import-page-with-different-ref-types
   (let [block-uuid (random-uuid)
@@ -186,16 +195,16 @@
           {:page {:build/journal 20250207 :block/uuid journal-uuid :build/keep-uuid? true}}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn)
-        full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
+        imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
 
-    (is (= (:properties original-data) (:properties full-imported-page))
+    (is (= (:properties original-data) (:properties imported-page))
         "Page's properties are imported")
-    (is (= (:classes original-data) (:classes full-imported-page))
+    (is (= (:classes original-data) (:classes imported-page))
         "Page's classes are imported")
-    (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page))
+    (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page))
         "Page's blocks are imported")
 
-    (import-second-time-appends-blocks conn conn2 "page1" original-data)))
+    (import-second-time-assertions conn conn2 "page1" original-data)))
 
 (deftest import-page-with-different-page-and-classes
   (let [original-data
@@ -210,16 +219,31 @@
            :blocks []}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn)
-        full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
+        imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
 
-    (is (= (:properties original-data) (:properties full-imported-page))
+    (is (= (:properties original-data) (:properties imported-page))
         "Page's properties are imported")
-    (is (= (:classes original-data) (:classes full-imported-page))
+    (is (= (:classes original-data) (:classes imported-page))
         "Page's classes are imported")
-    (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page))
+    (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page))
+        "Page's blocks are imported")
+
+    (import-second-time-assertions conn conn2 "page1" original-data)))
+
+(deftest import-journal-page
+  (let [original-data
+        {:pages-and-blocks
+         [{:page {:build/journal 20250210 :build/tags [:logseq.class/Journal]}
+           :blocks [{:block/title "b1"} {:block/title "b2"}]}]}
+        conn (db-test/create-conn-with-blocks original-data)
+        conn2 (db-test/create-conn)
+        journal-title (date-time-util/int->journal-title 20250210 "MMM do, yyyy")
+        imported-page (export-page-and-import-to-another-graph conn conn2 journal-title)]
+
+    (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page))
         "Page's blocks are imported")
 
-    (import-second-time-appends-blocks conn conn2 "page1" original-data)))
+    (import-second-time-assertions conn conn2 journal-title original-data)))
 
 (deftest import-page-with-different-property-types
   (let [block-object-uuid (random-uuid)
@@ -257,13 +281,13 @@
                      :build/keep-uuid? true}]}]}
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn)
-        full-imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
+        imported-page (export-page-and-import-to-another-graph conn conn2 "page1")]
 
-    (is (= (:properties original-data) (:properties full-imported-page))
+    (is (= (:properties original-data) (:properties imported-page))
         "Page's properties are imported")
-    (is (= (:classes original-data) (:classes full-imported-page))
+    (is (= (:classes original-data) (:classes imported-page))
         "Page's classes are imported")
-    (is (= (:pages-and-blocks original-data) (:pages-and-blocks full-imported-page))
+    (is (= (:pages-and-blocks original-data) (:pages-and-blocks imported-page))
         "Page's blocks are imported")))
 
 (deftest import-graph-ontology
@@ -292,8 +316,8 @@
         conn (db-test/create-conn-with-blocks original-data)
         conn2 (db-test/create-conn)
         {:keys [init-tx block-props-tx] :as _txs}
-        (->> (sqlite-export/build-graph-ontology-export @conn)
-             (sqlite-export/build-import @conn2 {}))
+        (-> (sqlite-export/build-graph-ontology-export @conn)
+            (sqlite-export/build-import @conn2 {}))
         ;; _ (cljs.pprint/pprint _txs)
         _ (d/transact! conn2 init-tx)
         _ (d/transact! conn2 block-props-tx)

src/main/frontend/handler/common/developer.cljs

@@ -69,7 +69,7 @@
 
 (defn- import-submit [import-inputs _e]
   (let [export-map (try (edn/read-string (:import-data @import-inputs)) (catch :default _err ::invalid-import))
-        import-block? (:build/block export-map)
+        import-block? (::sqlite-export/block export-map)
         block (when import-block?
                 (if-let [eid (:block-id (first (state/get-editor-args)))]
                   (db/entity [:block/uuid eid])
@@ -77,9 +77,9 @@
     (if (= ::invalid-import export-map)
       (notification/show! "The submitted EDN data is invalid! Fix and try again." :warning)
       (let [{:keys [init-tx block-props-tx error] :as txs}
-            (sqlite-export/build-import (db/get-db)
-                                        (when block {:current-block block})
-                                        export-map)]
+            (sqlite-export/build-import export-map
+                                        (db/get-db)
+                                        (when block {:current-block block}))]
         (pprint/pprint txs)
         (if error
           (notification/show! error :error)