enhance: make build-db-initial-data idempotent (#11750)

* enhance: make build-db-initial-data idempotent (1)

handle logseq/{config.edn,custom.css,custom.js}

* test: add build-db-initial-data-test

deps/common/src/logseq/common/uuid.cljs

@@ -19,23 +19,33 @@ the remaining chars for data of this type"
   (let [s-length (count s)]
     (apply str s (repeat (- length s-length) "0"))))
 
-(defn- gen-db-ident-block-uuid
-  "00000002-<hash-of-db-ident>-<padding-with-0>"
-  [db-ident]
-  {:pre [(keyword? db-ident)]}
-  (let [hash-num (str (Math/abs (hash db-ident)))
+(defn- gen-block-uuid
+  "prefix-<hash-of-db-ident>-<padding-with-0>"
+  [k prefix]
+  (let [hash-num (str (Math/abs (hash k)))
         part1 (fill-with-0 (subs hash-num 0 4) 4)
         part2 (fill-with-0 (subs hash-num 4 8) 4)
         part3 (fill-with-0 (subs hash-num 8 12) 4)
         part4 (fill-with-0 (subs hash-num 12) 12)]
-    (uuid (str "00000002-" part1 "-" part2 "-" part3 "-" part4))))
+    (uuid (str prefix "-" part1 "-" part2 "-" part3 "-" part4))))
+
+(defn- gen-db-ident-block-uuid
+  "00000002-<hash-of-db-ident>-<padding-with-0>"
+  [db-ident]
+  {:pre [(keyword? db-ident)]}
+  (gen-block-uuid db-ident "00000002"))
 
 (defn gen-uuid
   "supported type:
-  - :journal-page-uuid
-  - :db-ident-block-uuid"
+  - :journal-page-uuid, 00000001
+  - :db-ident-block-uuid, 00000002
+  - :migrate-new-block-uuid, 00000003
+  - :builtin-block-uuid, 00000004"
   ([] (d/squuid))
   ([type' v]
+   (assert (some? v))
    (case type'
      :journal-page-uuid (gen-journal-page-uuid v)
-     :db-ident-block-uuid (gen-db-ident-block-uuid v))))
+     :db-ident-block-uuid (gen-db-ident-block-uuid v)
+     :migrate-new-block-uuid (gen-block-uuid v "00000003")
+     :builtin-block-uuid (gen-block-uuid v "00000004"))))

deps/db/src/logseq/db/frontend/property/build.cljs

@@ -1,7 +1,7 @@
 (ns logseq.db.frontend.property.build
   "Builds core property concepts"
-  (:require [datascript.core :as d]
-            [logseq.common.util :as common-util]
+  (:require [logseq.common.util :as common-util]
+            [logseq.common.uuid :as common-uuid]
             [logseq.db.common.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
@@ -68,10 +68,10 @@
 (defn- build-property-value-block
   "Builds a property value entity given a block map/entity, a property entity or
   ident and its property value"
-  [block property value]
+  [block property value & {:keys [block-uuid]}]
   (let [block-id (or (:db/id block) (:db/ident block))]
     (-> (merge
-         {:block/uuid (d/squuid)
+         {:block/uuid (or block-uuid (common-uuid/gen-uuid))
           :block/page (if (:block/page block)
                         (:db/id (:block/page block))
                         ;; page block
@@ -91,18 +91,32 @@
   transacted, given a block and a properties map with raw property values. The
   properties map can have keys that are db-idents or they can be maps. If a map,
   it should have :original-property-id and :db/ident keys.  See
-  ->property-value-tx-m for such an example"
-  [block properties]
+  ->property-value-tx-m for such an example
+
+  :pure? - ensure this fn is a pure function"
+  [block properties & {:keys [pure?]}]
   ;; Build :db/id out of uuid if block doesn't have one for tx purposes
   (let [block' (if (:db/id block) block (assoc block :db/id [:block/uuid (:block/uuid block)]))]
     (->> properties
          (map (fn [[k v]]
-                (let [property-map (if (map? k) k {:db/ident k})]
+                (let [property-map (if (map? k) k {:db/ident k})
+                      gen-uuid-value-prefix (when pure?
+                                              (or (:db/ident block) (:block/uuid block)))]
                   (assert (:db/ident property-map) "Key in map must have a :db/ident")
+                  (when pure? (assert (some? gen-uuid-value-prefix) block))
                   [(or (:original-property-id property-map) (:db/ident property-map))
                    (if (set? v)
-                     (set (map #(build-property-value-block block' property-map %) v))
-                     (build-property-value-block block' property-map v))])))
+                     (set (map #(build-property-value-block
+                                 block' property-map %
+                                 (when pure?
+                                   {:block-uuid
+                                    (common-uuid/gen-uuid :builtin-block-uuid (str gen-uuid-value-prefix "-" %))}))
+                               v))
+                     (build-property-value-block block' property-map v
+                                                 (when pure?
+                                                   {:block-uuid
+                                                    (common-uuid/gen-uuid
+                                                     :builtin-block-uuid (str gen-uuid-value-prefix "-" v))})))])))
          (into {}))))
 
 (defn build-properties-with-ref-values

deps/db/src/logseq/db/sqlite/create_graph.cljs

@@ -1,14 +1,13 @@
 (ns logseq.db.sqlite.create-graph
   "Helper fns for creating a DB graph"
   (:require [clojure.string :as string]
-            [datascript.core :as d]
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
+            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.entity-util :as entity-util]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
-            [logseq.db.common.order :as db-order]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.build :as db-property-build]
             [logseq.db.frontend.property.type :as db-property-type]
@@ -32,20 +31,22 @@
   "Given a new block and its properties, creates a map of properties which have values of property value tx.
    This map is used for both creating the new property values and then adding them to a block"
   [new-block properties]
-  (->> properties
-       (keep (fn [[k v]]
-               (when-let [built-in-type (get-in db-property/built-in-properties [k :schema :type])]
-                 (if (and (db-property-type/value-ref-property-types built-in-type)
-                          ;; closed values are referenced by their :db/ident so no need to create values
-                          (not (get-in db-property/built-in-properties [k :closed-values])))
-                   (let [property-map {:db/ident k
-                                       :logseq.property/type built-in-type}]
-                     [property-map v])
-                   (when-let [built-in-type' (get (:build/properties-ref-types new-block) built-in-type)]
-                     (let [property-map {:db/ident k
-                                         :logseq.property/type built-in-type'}]
-                       [property-map v]))))))
-       (db-property-build/build-property-values-tx-m new-block)))
+  (db-property-build/build-property-values-tx-m
+   new-block
+   (->> properties
+        (keep (fn [[k v]]
+                (when-let [built-in-type (get-in db-property/built-in-properties [k :schema :type])]
+                  (if (and (db-property-type/value-ref-property-types built-in-type)
+                           ;; closed values are referenced by their :db/ident so no need to create values
+                           (not (get-in db-property/built-in-properties [k :closed-values])))
+                    (let [property-map {:db/ident k
+                                        :logseq.property/type built-in-type}]
+                      [property-map v])
+                    (when-let [built-in-type' (get (:build/properties-ref-types new-block) built-in-type)]
+                      (let [property-map {:db/ident k
+                                          :logseq.property/type built-in-type'}]
+                        [property-map v])))))))
+   :pure? true))
 
 (defn build-properties
   "Given a properties map in the format of db-property/built-in-properties, builds their properties tx"
@@ -170,7 +171,7 @@
 (defn build-initial-views
   "Builds initial blocks used for storing views. Used by db and file graphs"
   []
-  (let [page-id (common-uuid/gen-uuid)]
+  (let [page-id (common-uuid/gen-uuid :builtin-block-uuid common-config/views-page-name)]
     [(sqlite-util/block-with-timestamps
       {:block/uuid page-id
        :block/name common-config/views-page-name
@@ -179,7 +180,7 @@
        :logseq.property/hide? true
        :logseq.property/built-in? true})
      (sqlite-util/block-with-timestamps
-      {:block/uuid (common-uuid/gen-uuid)
+      {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "All Pages Default View")
        :block/title "All Pages Default View"
        :block/parent [:block/uuid page-id]
        :block/order (db-order/gen-key nil)
@@ -190,7 +191,7 @@
 (defn- build-favorites-page
   []
   [(sqlite-util/block-with-timestamps
-    {:block/uuid (common-uuid/gen-uuid)
+    {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid common-config/favorites-page-name)
      :block/name common-config/favorites-page-name
      :block/title common-config/favorites-page-name
      :block/tags [:logseq.class/Page]
@@ -211,17 +212,17 @@
                        {:db/ident :logseq.property/empty-placeholder}]
                        import-type
                        (into (sqlite-util/import-tx import-type)))
-        initial-files [{:block/uuid (d/squuid)
+        initial-files [{:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/config.edn")
                         :file/path (str "logseq/" "config.edn")
                         :file/content config-content
                         :file/created-at (js/Date.)
                         :file/last-modified-at (js/Date.)}
-                       {:block/uuid (d/squuid)
+                       {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.css")
                         :file/path (str "logseq/" "custom.css")
                         :file/content ""
                         :file/created-at (js/Date.)
                         :file/last-modified-at (js/Date.)}
-                       {:block/uuid (d/squuid)
+                       {:block/uuid (common-uuid/gen-uuid :builtin-block-uuid "logseq/custom.js")
                         :file/path (str "logseq/" "custom.js")
                         :file/content ""
                         :file/created-at (js/Date.)

deps/db/src/logseq/db/sqlite/util.cljs

@@ -3,16 +3,16 @@
   (:require [cljs-bean.transit]
             [clojure.string :as string]
             [cognitect.transit :as transit]
-            [datascript.core :as d]
+            [datascript.core]
             [datascript.impl.entity :as de]
             [datascript.transit :as dt]
             [logseq.common.util :as common-util]
             [logseq.common.uuid :as common-uuid]
             [logseq.db.common.order :as db-order]
+            [logseq.db.file-based.schema :as file-schema]
             [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.property.type :as db-property-type]
-            [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.file-based.schema :as file-schema]))
+            [logseq.db.frontend.schema :as db-schema]))
 
 (defonce db-version-prefix "logseq_db_")
 (defonce file-version-prefix "logseq_local_")
@@ -121,7 +121,7 @@
   (block-with-timestamps
    {:block/name (common-util/page-name-sanity-lc page-name)
     :block/title page-name
-    :block/uuid (d/squuid)
+    :block/uuid (common-uuid/gen-uuid :builtin-block-uuid page-name)
     :block/tags #{:logseq.class/Page}}))
 
 (defn kv

deps/db/test/logseq/db/sqlite/create_graph_test.cljs

@@ -1,5 +1,6 @@
 (ns logseq.db.sqlite.create-graph-test
   (:require [cljs.test :refer [deftest is testing]]
+            [clojure.data :as data]
             [clojure.set :as set]
             [clojure.string :as string]
             [datascript.core :as d]
@@ -150,3 +151,21 @@
 
       (is (empty? (map :entity (:errors (db-validate/validate-db! @conn))))
           "Graph with different :url blocks has no validation errors"))))
+
+(deftest build-db-initial-data-test
+  (testing "idempotent initial-data"
+    (letfn [(remove-ignored-attrs&entities [init-data]
+              (let [[before after] (split-with #(not= :logseq.kv/graph-created-at (:db/ident %)) init-data)
+                    init-data* (concat before (rest after))]
+                (map (fn [ent] (dissoc ent
+                                       :block/created-at :block/updated-at
+                                       :file/last-modified-at :file/created-at
+                                       :block/order ;; TODO: block/order should be same as well
+                                       ))
+                     init-data*)))]
+      (let [[first-only second-only common]
+            (data/diff (remove-ignored-attrs&entities (sqlite-create-graph/build-db-initial-data ""))
+                       (remove-ignored-attrs&entities (sqlite-create-graph/build-db-initial-data "")))]
+        (is (and (every? nil? first-only)
+                 (every? nil? second-only))
+            (pr-str [first-only second-only common]))))))