graph export optionally exports timestamps

deps/db/src/logseq/db/sqlite/export.cljs

@@ -83,7 +83,7 @@
 
 (defn- build-export-properties
   "The caller of this fn is responsible for building :build/:property-classes unless shallow-copy?"
-  [db user-property-idents {:keys [include-properties? include-uuid? shallow-copy?]}]
+  [db user-property-idents {:keys [include-properties? include-timestamps? include-uuid? shallow-copy?]}]
   (let [properties-config-by-ent
         (->> user-property-idents
              (map (fn [ident]
@@ -95,6 +95,8 @@
                                                 (conj :block/title)))
                          include-uuid?
                          (assoc :block/uuid (:block/uuid property))
+                         include-timestamps?
+                         (merge (select-keys property [:block/created-at :block/updated-at]))
                          (and (not shallow-copy?) (:logseq.property/classes property))
                          (assoc :build/property-classes (mapv :db/ident (:logseq.property/classes property)))
                          (seq closed-values)
@@ -123,11 +125,13 @@
 (defn- build-export-class
   "The caller of this fn is responsible for building any classes or properties from this fn
    unless shallow-copy?"
-  [class-ent {:keys [include-parents? include-uuid? shallow-copy?]
+  [class-ent {:keys [include-parents? include-uuid? shallow-copy? include-timestamps?]
               :or {include-parents? true}}]
   (cond-> (select-keys class-ent [:block/title])
     include-uuid?
     (assoc :block/uuid (:block/uuid class-ent))
+    include-timestamps?
+    (merge (select-keys class-ent [:block/created-at :block/updated-at]))
     (and (:logseq.property.class/properties class-ent) (not shallow-copy?))
     (assoc :build/class-properties
            (mapv :db/ident (:logseq.property.class/properties class-ent)))
@@ -167,7 +171,7 @@
           (into {})))))
 
 (defn- build-node-properties
-  [db entity ent-properties properties]
+  [db entity ent-properties {:keys [properties] :as options}]
   (let [new-user-property-ids (->> (keys ent-properties)
                                    (concat (->> (:block/tags entity)
                                                 (mapcat :logseq.property.class/properties)
@@ -175,22 +179,26 @@
                                    ;; Built-in properties and any possible modifications are not exported
                                    (remove db-property/logseq-property?)
                                    (remove #(get properties %)))]
-    ;; Classes from hare are built in build-node-classes
-    (build-export-properties db new-user-property-ids {})))
+    ;; Classes from here are built in build-node-classes
+    (build-export-properties db new-user-property-ids options)))
 
 (defn- build-node-export
   "Given a block/page entity and optional existing properties, build an export map of its
    tags and properties"
-  [db entity {:keys [properties include-uuid-fn shallow-copy?]
-              :or {include-uuid-fn (constantly false)}}]
+  [db entity {:keys [properties include-uuid-fn shallow-copy? include-timestamps?]
+              :or {include-uuid-fn (constantly false)}
+              :as options}]
   (let [ent-properties (dissoc (db-property/properties entity) :block/tags)
         build-tags (when (seq (:block/tags entity)) (->build-tags (:block/tags entity)))
-        new-properties (when-not shallow-copy? (build-node-properties db entity ent-properties properties))
+        new-properties (when-not shallow-copy?
+                         (build-node-properties db entity ent-properties (select-keys options [:properties :include-timestamps?])))
         build-node (cond-> {:block/title (block-title entity)}
                      (:block/link entity)
                      (assoc :block/link [:block/uuid (:block/uuid (:block/link entity))])
                      (include-uuid-fn (:block/uuid entity))
                      (assoc :block/uuid (:block/uuid entity) :build/keep-uuid? true)
+                     include-timestamps?
+                     (merge (select-keys entity [:block/created-at :block/updated-at]))
                      (and (not shallow-copy?) (seq build-tags))
                      (assoc :build/tags build-tags)
                      (and (not shallow-copy?) (seq ent-properties))
@@ -365,7 +373,7 @@
            block-export)))
 
 (defn- build-page-blocks-export [db page-entity {:keys [properties classes blocks] :as opts}]
-  (let [page-ent-export (build-node-export db page-entity (select-keys opts [:properties :include-uuid-fn]))
+  (let [page-ent-export (build-node-export db page-entity (select-keys opts [:properties :include-uuid-fn :include-timestamps?]))
         page (merge (dissoc (:node page-ent-export) :block/title)
                     (shallow-copy-page page-entity))
         page-blocks-export {:pages-and-blocks [{:page page :blocks blocks}]
@@ -379,15 +387,15 @@
        (map #(d/entity db %))))
 
 (defn- build-page-export*
-  [db eid page-blocks* include-uuid-fn]
+  [db eid page-blocks* options]
   (let [page-entity (d/entity db eid)
         page-blocks (->> page-blocks*
                          (sort-by :block/order)
                          ;; Remove property value blocks as they are exported in a block's :build/properties
                          (remove #(:logseq.property/created-from-property %)))
         {:keys [pvalue-uuids] :as blocks-export}
-        (build-blocks-export db page-blocks {:include-uuid-fn include-uuid-fn})
-        page-blocks-export (build-page-blocks-export db page-entity (assoc blocks-export :include-uuid-fn include-uuid-fn))
+        (build-blocks-export db page-blocks options)
+        page-blocks-export (build-page-blocks-export db page-entity (merge blocks-export options))
         page-export (assoc page-blocks-export :pvalue-uuids pvalue-uuids)]
     page-export))
 
@@ -396,7 +404,8 @@
   [db eid]
   (let [page-blocks* (get-page-blocks db eid)
         {:keys [content-ref-ents] :as content-ref-export} (build-content-ref-export db page-blocks*)
-        {:keys [pvalue-uuids] :as page-export*} (build-page-export* db eid page-blocks* (:content-ref-uuids content-ref-export))
+        {:keys [pvalue-uuids] :as page-export*}
+        (build-page-export* db eid page-blocks* {:include-uuid-fn (:content-ref-uuids content-ref-export)})
         page-entity (d/entity db eid)
         uuid-block-export (build-uuid-block-export db pvalue-uuids content-ref-ents {:page-entity page-entity})
         page-export (finalize-export-maps db page-export* uuid-block-export content-ref-export)]
@@ -444,13 +453,13 @@
 
 (defn- build-graph-ontology-export
   "Exports a graph's tags and properties"
-  [db]
+  [db options]
   (let [user-property-idents (d/q '[:find [?db-ident ...]
                                     :where [?p :db/ident ?db-ident]
                                     [?p :block/tags :logseq.class/Property]
                                     (not [?p :logseq.property/built-in?])]
                                   db)
-        properties (build-export-properties db user-property-idents {:include-properties? true})
+        properties (build-export-properties db user-property-idents (merge options {:include-properties? true}))
         class-ents (->> (d/q '[:find [?class ...]
                                :where [?class :block/tags :logseq.class/Tag]
                                (not [?class :logseq.property/built-in?])]
@@ -461,7 +470,7 @@
              (map (fn [ent]
                     (let [ent-properties (dissoc (db-property/properties ent) :block/tags :logseq.property/parent)]
                       (vector (:db/ident ent)
-                              (cond-> (build-export-class ent {})
+                              (cond-> (build-export-class ent options)
                                 (seq ent-properties)
                                 (assoc :build/properties (buildable-properties db ent-properties properties)))))))
              (into {}))]
@@ -484,13 +493,13 @@
 
 (defn- build-graph-pages-export
   "Handles pages, journals and their blocks"
-  [db _options]
+  [db options]
   (let [page-ids (concat (map :e (d/datoms db :avet :block/tags :logseq.class/Page))
                          (map :e (d/datoms db :avet :block/tags :logseq.class/Journal)))
         content-ref-uuids (get-graph-content-ref-uuids db)
         page-exports (mapv (fn [eid]
                              (let [page-blocks* (get-page-blocks db eid)]
-                               (build-page-export* db eid page-blocks* (constantly true))))
+                               (build-page-export* db eid page-blocks* (merge options {:include-uuid-fn (constantly true)}))))
                            page-ids)
         all-ref-uuids (set/union content-ref-uuids (set (mapcat :pvalue-uuids page-exports)))
         pages-export (apply merge-export-maps page-exports)
@@ -507,14 +516,20 @@
     pages-export'))
 
 (defn- build-graph-files
-  [db]
-  (->> (d/q '[:find [(pull ?b [*]) ...] :where [?b :file/path]] db)
-       (mapv #(select-keys % [:file/path :file/content]))))
-
-(defn- build-graph-export [db options]
-  (let [ontology-export (build-graph-ontology-export db)
-        pages-export (build-graph-pages-export db options)
-        files (build-graph-files db)
+  [db {:keys [include-timestamps?]}]
+  (->> (d/q '[:find [(pull ?b [:file/path :file/content :file/created-at :file/last-modified-at]) ...]
+              :where [?b :file/path]] db)
+       (mapv #(if include-timestamps?
+                (select-keys % [:file/path :file/content :file/created-at :file/last-modified-at])
+                (select-keys % [:file/path :file/content])))))
+
+(defn- build-graph-export
+  "Exports whole graph. Has the following options:
+   * :include-timestamps? - When set timestamps are included on all blocks"
+  [db options]
+  (let [ontology-export (build-graph-ontology-export db (select-keys options [:include-timestamps?]))
+        pages-export (build-graph-pages-export db (select-keys options [:include-timestamps?]))
+        files (build-graph-files db options)
         graph-export (merge (merge-export-maps ontology-export pages-export)
                             {::graph-files files})]
     graph-export))
@@ -585,7 +600,7 @@
           :view-nodes
           (build-view-nodes-export db (:node-ids options))
           :graph-ontology
-          (build-graph-ontology-export db)
+          (build-graph-ontology-export db {})
           :graph
           (build-graph-export db (:graph-options options)))]
     (ensure-export-is-valid (dissoc export-map ::block ::graph-files))

deps/db/test/logseq/db/sqlite/export_test.cljs

@@ -2,6 +2,7 @@
   (:require [cljs.pprint]
             [cljs.test :refer [deftest is testing]]
             [datascript.core :as d]
+            [logseq.common.config :as common-config]
             [logseq.common.util.date-time :as date-time-util]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
@@ -9,7 +10,8 @@
             [logseq.db.sqlite.export :as sqlite-export]
             [logseq.db.test.helper :as db-test]
             [medley.core :as medley]
-            [logseq.common.config :as common-config]))
+            [clojure.walk :as walk]
+            [logseq.common.util :as common-util]))
 
 ;; Test helpers
 ;; ============
@@ -245,7 +247,7 @@
 
     (import-second-time-assertions conn conn2 "page1" original-data)))
 
-(deftest ^:focus2 import-page-with-different-ref-types
+(deftest ^:focus3 import-page-with-different-ref-types
   (let [block-uuid (random-uuid)
         internal-block-uuid (random-uuid)
         class-uuid (random-uuid)
@@ -510,7 +512,8 @@
     (is (= (expand-properties (:properties original-data)) (:properties imported-nodes)))
     (is (= (expand-classes (:classes original-data)) (:classes imported-nodes)))))
 
-(deftest ^:focus import-graph
+(defn- build-original-graph-data
+  []
   (let [internal-block-uuid (random-uuid)
         favorited-uuid (random-uuid)
         block-object-uuid (random-uuid)
@@ -571,32 +574,84 @@
                      {:logseq.property.view/type :logseq.property.view/type.list,
                       :logseq.property.view/feature-type :linked-references,
                       :logseq.property/view-for
-                      [:build/page {:build/journal 19650201}]}}]}]}
-        original-files
-        [{:file/path "logseq/config.edn"
-          :file/content "{:foo :bar}"}
-         {:file/path "logseq/custom.css"
-          :file/content ".foo {background-color: blue}"}
-         {:file/path "logseq/custom.js"
-          :file/content "// comment"}]
-        conn (db-test/create-conn-with-blocks original-data)
-        _ (d/transact! conn original-files)
-        conn2 (db-test/create-conn)
-        {:keys [init-tx block-props-tx misc-tx] :as _txs}
-        (-> (sqlite-export/build-export @conn {:export-type :graph})
-            (sqlite-export/build-import @conn2 {}))
+                      [:build/page {:build/journal 19650201}]}}]}]
+         ::sqlite-export/graph-files
+         [{:file/path "logseq/config.edn"
+           :file/content "{:foo :bar}"}
+          {:file/path "logseq/custom.css"
+           :file/content ".foo {background-color: blue}"}
+          {:file/path "logseq/custom.js"
+           :file/content "// comment"}]}]
+    original-data))
+
+(defn- export-graph-and-import-to-another-graph
+  "Exports graph and imports it to a 2nd graph, validates it and then exports the 2nd graph"
+  [export-conn import-conn export-options]
+  (let [{:keys [init-tx block-props-tx misc-tx] :as _txs}
+        (-> (sqlite-export/build-export @export-conn {:export-type :graph :graph-options export-options})
+            (sqlite-export/build-import @import-conn {}))
         ;; _ (cljs.pprint/pprint _txs)
-        _ (d/transact! conn2 init-tx)
-        _ (d/transact! conn2 block-props-tx)
-        _ (d/transact! conn2 misc-tx)
-        _ (validate-db @conn2)
-        imported-graph (sqlite-export/build-export @conn2 {:export-type :graph})]
+        _ (d/transact! import-conn init-tx)
+        _ (d/transact! import-conn block-props-tx)
+        _ (d/transact! import-conn misc-tx)
+        _ (validate-db @import-conn)
+        imported-graph (sqlite-export/build-export @import-conn {:export-type :graph :graph-options export-options})]
+    imported-graph))
+
+(deftest ^:focus import-graph
+  (let [original-data (build-original-graph-data)
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn)
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {})]
+
+    ;; (cljs.pprint/pprint (set (:pages-and-blocks original-data)))
+    ;; (cljs.pprint/pprint (set (:pages-and-blocks imported-graph)))
+    (is (= (set (:pages-and-blocks original-data)) (set (:pages-and-blocks imported-graph))))
+    (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
+    (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
+        "All :file/path entities are imported")))
+
+(deftest ^:focus2 import-graph-with-timestamps
+  (let [original-data* (build-original-graph-data)
+        original-data (-> original-data*
+                          (update :pages-and-blocks
+                                  (fn [pages-and-blocks]
+                                    (walk/postwalk (fn [e]
+                                                     (cond
+                                                       (and (map? e) (or (:block/title e) (:build/journal e)))
+                                                       (common-util/block-with-timestamps e)
+                                                       ;; Don't add timestamps to pvalues
+                                                       (and (vector? e) (= :build/page (first e)))
+                                                       [(first e) (dissoc (second e) :block/updated-at :block/created-at)]
+                                                       :else
+                                                       e))
+                                                   pages-and-blocks)))
+                          (update :classes update-vals common-util/block-with-timestamps)
+                          (update :properties update-vals common-util/block-with-timestamps)
+                          (update ::sqlite-export/graph-files
+                                  (fn [files]
+                                    (mapv #(let [now (js/Date.)]
+                                             (merge % {:file/created-at now :file/last-modified-at now}))
+                                          files))))
+        conn (db-test/create-conn-with-blocks (dissoc original-data ::sqlite-export/graph-files))
+        _ (d/transact! conn (::sqlite-export/graph-files original-data))
+        conn2 (db-test/create-conn)
+        imported-graph (export-graph-and-import-to-another-graph conn conn2 {:include-timestamps? true})
+        ignore-timestamps-if-built-in
+        (fn [m]
+          (if (get-in m [:page :build/properties :logseq.property/built-in?])
+            (update m :page dissoc :block/created-at :block/updated-at)
+            m))]
 
     ;; (cljs.pprint/pprint (set (:pages-and-blocks original-data)))
     ;; (cljs.pprint/pprint (set (:pages-and-blocks imported-graph)))
-    (is (= (set (:pages-and-blocks original-data))
-           (set (:pages-and-blocks imported-graph))))
+    ;; (cljs.pprint/pprint (butlast (clojure.data/diff (set (map ignore-timestamps-if-built-in (:pages-and-blocks original-data)))
+    ;;                                                 (set (map ignore-timestamps-if-built-in (:pages-and-blocks imported-graph))))))
+    (is (= (set (map ignore-timestamps-if-built-in (:pages-and-blocks original-data)))
+           (set (map ignore-timestamps-if-built-in (:pages-and-blocks imported-graph)))))
     (is (= (expand-properties (:properties original-data)) (:properties imported-graph)))
     (is (= (expand-classes (:classes original-data)) (:classes imported-graph)))
-    (is (= original-files (::sqlite-export/graph-files imported-graph))
+    (is (= (::sqlite-export/graph-files original-data) (::sqlite-export/graph-files imported-graph))
         "All :file/path entities are imported")))