enhance: add :logseq.kv data to graph export

Addresses
https://github.com/logseq/logseq/pull/11784#pullrequestreview-2697690952.
Also improves :set option and removes outdated code in a test

deps/db/script/diff_graphs.cljs

@@ -54,11 +54,15 @@
            (-> m
                (update :classes update-vals (fn [m]
                                               (update m :build/class-properties sort)))
+               (update ::sqlite-export/kv-values
+                       (fn [kvs]
+                         ;; Ignore extra metadata that a copied graph can add
+                         (vec (remove #(#{:logseq.kv/import-type :logseq.kv/imported-at} (:db/ident %)) kvs))))
               ;; TODO: fix built-in views for schema export
                (update :pages-and-blocks (fn [pbs]
                                            (vec (remove #(= (:block/title (:page %)) common-config/views-page-name) pbs)))))
             (:set-diff options)
-            (update :pages-and-blocks set)))
+            (update-vals set)))
         diff (->> (data/diff (prepare-export-to-diff export-map) (prepare-export-to-diff export-map2))
                   butlast)]
     (if (= diff [nil nil])

deps/db/src/logseq/db/sqlite/export.cljs

@@ -478,8 +478,8 @@
                                     (not [?p :logseq.property/built-in?])]
                                   db)
         user-property-idents' (if (seq exclude-namespaces)
-                               (remove #(re-find exclude-regex (namespace %)) user-property-idents)
-                               user-property-idents)
+                                (remove #(re-find exclude-regex (namespace %)) user-property-idents)
+                                user-property-idents)
         properties (build-export-properties db user-property-idents' (merge options {:include-properties? true}))
         class-ents (->> (d/q '[:find [?class ...]
                                :where [?class :block/tags :logseq.class/Tag]
@@ -550,6 +550,14 @@
                 (select-keys % [:file/path :file/content :file/created-at :file/last-modified-at])
                 (select-keys % [:file/path :file/content])))))
 
+(defn- build-kv-values
+  [db]
+  (->> (d/q '[:find [(pull ?b [:db/ident :kv/value]) ...]
+              :where [?b :kv/value]] db)
+       ;; Don't export schema-version as frontend sets this and shouldn't be overridden
+       (remove #(= :logseq.kv/schema-version (:db/ident %)))
+       vec))
+
 (defn remove-uuids-if-not-ref [export-map all-ref-uuids]
   (let [remove-uuid-if-not-ref (fn [m] (if (contains? all-ref-uuids (:block/uuid m))
                                          m
@@ -609,12 +617,14 @@
                        graph-export*)
         all-ref-uuids (set/union content-ref-uuids ontology-pvalue-uuids (:pvalue-uuids pages-export))
         files (build-graph-files db options)
+        kv-values (build-kv-values db)
         ;; Remove all non-ref uuids after all nodes are built.
         ;; Only way to ensure all pvalue uuids present across block types
         graph-export' (-> (remove-uuids-if-not-ref graph-export all-ref-uuids)
                           (update :pages-and-blocks sort-pages-and-blocks))]
     (merge graph-export'
-           {::graph-files files})))
+           {::graph-files files
+            ::kv-values kv-values})))
 
 (defn- find-undefined-classes-and-properties [{:keys [classes properties pages-and-blocks]}]
   (let [referenced-classes
@@ -688,7 +698,7 @@
           (build-graph-ontology-export db {})
           :graph
           (build-graph-export db (:graph-options options)))]
-    (ensure-export-is-valid (dissoc export-map ::block ::graph-files) options)
+    (ensure-export-is-valid (dissoc export-map ::block ::graph-files ::kv-values) options)
     export-map))
 
 ;; Import fns
@@ -709,7 +719,7 @@
 (defn- check-for-existing-entities
   "Checks export map for existing entities and adds :block/uuid to them if they exist in graph to import.
    Also checks for property conflicts between existing properties and properties to be imported"
-  [db {:keys [pages-and-blocks classes properties] ::keys [graph-files]} property-conflicts]
+  [db {:keys [pages-and-blocks classes properties] ::keys [graph-files kv-values]} property-conflicts]
   (let [export-map
         (cond-> {:build-existing-tx? true
                  :extract-content-refs? false}
@@ -741,6 +751,8 @@
                                  [k (assoc v :block/uuid (:block/uuid ent))])
                                [k v])))
                       (into {})))
+          (seq kv-values)
+          (assoc ::kv-values kv-values)
           ;; Currently all files are created by app so no need to distinguish between user and built-in ones yet
           (seq graph-files)
           (assoc ::graph-files graph-files))
@@ -764,8 +776,15 @@
     (merge-export-maps export-map {:pages-and-blocks pages-and-blocks})))
 
 (defn build-import
-  "Given an entity's export map, build the import tx to create it. Returns a map
-   of txs to transact with the following keys:
+  "Given an entity's export map, build the import tx to create it. In addition to standard sqlite.build keys,
+   an export map can have the following namespaced keys:
+   * ::block - Block map for a :block export
+   * ::graph-files - Vec of files for a :graph export
+   * ::kv-values - Vec of :kv/value maps for a :graph export
+   * ::auto-include-namespaces - A set of parent namespaces to include from properties and classes
+     for a :graph export. See :exclude-namespaces in build-graph-export for a similar option
+
+   This fn then returns a map of txs to transact with the following keys:
    * :init-tx - Txs that must be transacted first, usually because they define new properties
    * :block-props-tx - Txs to transact after :init-tx, usually because they use newly defined properties
    * :misc-tx - Txs to transact unrelated to other txs"
@@ -774,7 +793,7 @@
                      (build-block-import-options current-block export-map*)
                      export-map*)
         export-map' (if (seq (::auto-include-namespaces export-map*))
-                      (merge (select-keys export-map [::graph-files])
+                      (merge (select-keys export-map [::graph-files ::kv-values])
                              (add-ontology-for-include-namespaces db export-map))
                       export-map)
         property-conflicts (atom [])
@@ -784,6 +803,9 @@
         (js/console.error :property-conflicts @property-conflicts)
         {:error (str "The following imported properties conflict with the current graph: "
                      (pr-str (mapv :property-id @property-conflicts)))})
-      (cond-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files))
-        (seq (::graph-files export-map''))
-        (assoc :misc-tx (::graph-files export-map''))))))
+      (if (or (::graph-files export-map') (::kv-values export-map'))
+        ;; only for :graph export-type
+        (-> (sqlite-build/build-blocks-tx (dissoc export-map'' ::graph-files ::kv-values))
+            (assoc :misc-tx (vec (concat (::graph-files export-map'')
+                                         (::kv-values export-map'')))))
+        (sqlite-build/build-blocks-tx export-map'')))))

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -643,6 +643,10 @@
 (deftest import-graph
   (let [original-data (build-original-graph-data)
         conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        ;; set to an unobtainable version to test this ident
+        _ (d/transact! conn [{:db/ident :logseq.kv/schema-version :kv/value {:major 1 :minor 0}}])
+        original-kv-values (remove #(= :logseq.kv/schema-version (:db/ident %))
+                                   (d/q '[:find [(pull ?b [:db/ident :kv/value]) ...] :where [?b :kv/value]] @conn))
         _ (d/transact! conn (::sqlite-export/graph-files original-data))
         conn2 (db-test/create-conn)
         imported-graph (export-graph-and-import-to-another-graph conn conn2 {})]
@@ -657,7 +661,12 @@
     (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
     (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
     (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
-        "All :file/path entities are imported")))
+        "All :file/path entities are imported")
+    (is (= original-kv-values (::sqlite-export/kv-values imported-graph))
+        "All :kv/value entities are imported except for ignored ones")
+    (is (not= (:kv/value (d/entity @conn :logseq.kv/schema-version))
+              (:kv/value (d/entity @conn2 :logseq.kv/schema-version)))
+        "Ignored :kv/value is not updated")))
 
 (deftest import-graph-with-timestamps
   (let [original-data* (build-original-graph-data)
@@ -665,13 +674,8 @@
                           (update :pages-and-blocks
                                   (fn [pages-and-blocks]
                                     (walk/postwalk (fn [e]
-                                                     (cond
-                                                       (and (map? e) (or (:block/title e) (:build/journal e)))
+                                                     (if (and (map? e) (or (:block/title e) (:build/journal e)))
                                                        (common-util/block-with-timestamps e)
-                                                       ;; Don't add timestamps to pvalues
-                                                       (and (vector? e) (= :build/page (first e)))
-                                                       [(first e) (dissoc (second e) :block/updated-at :block/created-at)]
-                                                       :else
                                                        e))
                                                    pages-and-blocks)))
                           (update :classes update-vals common-util/block-with-timestamps)

src/main/frontend/handler/db_based/import.cljs

@@ -1,6 +1,7 @@
 (ns frontend.handler.db-based.import
   "Handles DB graph imports"
   (:require [clojure.edn :as edn]
+            [cljs.pprint :as pprint]
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.handler.notification :as notification]
@@ -109,9 +110,9 @@
                   (notification/show! "No block found" :warning)))]
     (if (= ::invalid-import export-map)
       (notification/show! "The submitted EDN data is invalid! Please fix and try again." :warning)
-      (let [{:keys [init-tx block-props-tx misc-tx error] :as _txs}
+      (let [{:keys [init-tx block-props-tx misc-tx error] :as txs}
             (safe-build-edn-import export-map (when block {:current-block block}))]
-        ;; (cljs.pprint/pprint _txs)
+        (pprint/pprint txs)
         (if error
           (notification/show! error :error)
           ;; TODO: When not import-block, use metadata that supports undo