enhance: update :rename-db-idents type migration

deps/db/src/logseq/db/frontend/property.cljs

@@ -563,11 +563,7 @@
                                                                 :hide? true}
                                                        :rtc {:rtc/ignore-attr-when-init-upload true
                                                              :rtc/ignore-attr-when-init-download true
-                                                             :rtc/ignore-attr-when-syncing true}}
-     ;; TODO: remove later
-     :logseq.property/test1 {:title "test1 property"
-                             :schema {:type :default
-                                      :public? true}})))
+                                                             :rtc/ignore-attr-when-syncing true}})))
 
 (def db-attribute-properties
   "Internal properties that are also db schema attributes"

src/main/frontend/worker/db/migrate.cljs

@@ -290,12 +290,29 @@
                    db)]
     (mapcat
      (fn [id]
-       (let [title (:block/title (d/entity db id))]
-         [[:db/add id :db/ident (db-class/create-user-class-ident-from-name db title)]
-          [:db/add id :logseq.property.class/extends :logseq.class/Root]
-          [:db/retract id :block/tags :logseq.class/Page]
-          [:db/retract id :block/refs :logseq.class/Page]
-          [:db/retract id :block/path-refs :logseq.class/Page]]))
+       [[:db/add id :logseq.property.class/extends :logseq.class/Root]
+        [:db/retract id :block/tags :logseq.class/Page]
+        [:db/retract id :block/refs :logseq.class/Page]
+        [:db/retract id :block/path-refs :logseq.class/Page]])
+     class-ids)))
+
+(defn add-missing-db-ident-for-tags2
+  [db]
+  (let [class-ids
+        (d/q
+         '[:find [?b ...]
+           :where
+           [?b :block/tags :logseq.class/Tag]
+           [(missing? $ ?b :db/ident)]]
+         db)]
+    (keep
+     (fn [id]
+       (let [ent (d/entity db id)
+             title (:block/title ent)
+             block-uuid (:block/uuid ent)]
+         (when block-uuid
+           {:db-ident-or-block-uuid block-uuid
+            :new-db-ident (db-class/create-user-class-ident-from-name db title)})))
      class-ids)))
 
 (defn fix-using-properties-as-tags
@@ -312,11 +329,30 @@
                       (map first))]
     (mapcat
      (fn [id]
-       (let [property (d/entity db id)
-             title (:block/title property)]
-         (into (retract-property-attributes id)
-               [[:db/retract id :logseq.property/parent]
-                [:db/add id :db/ident (db-class/create-user-class-ident-from-name db title)]])))
+       (into (retract-property-attributes id)
+             [[:db/retract id :logseq.property/parent]]))
+     property-ids)))
+
+(defn fix-using-properties-as-tags2
+  [db]
+  (let [property-ids
+        (->>
+         (d/q
+          '[:find ?b ?i
+            :where
+            [?b :block/tags :logseq.class/Tag]
+            [?b :db/ident ?i]]
+          db)
+         (filter (fn [[_ ident]] (= "user.property" (namespace ident))))
+         (map first))]
+    (keep
+     (fn [id]
+       (let [ent (d/entity db id)
+             title (:block/title ent)
+             block-uuid (:block/uuid ent)]
+         (when block-uuid
+           {:db-ident-or-block-uuid block-uuid
+            :new-db-ident (db-class/create-user-class-ident-from-name db title)})))
      property-ids)))
 
 (defn remove-block-order-for-tags
@@ -358,20 +394,17 @@
 (def schema-version->updates
   "A vec of tuples defining datascript migrations. Each tuple consists of the
    schema version integer and a migration map. A migration map can have keys of :properties, :classes
-   and :fix."
+   :rename-db-idents and :fix."
   [["65.0" {:fix separate-classes-and-properties}]
    ["65.1" {:fix fix-rename-parent-to-extends}]
    ["65.2" {:fix fix-tag-properties}]
-   ["65.3" {:fix add-missing-db-ident-for-tags}]
-   ["65.4" {:fix fix-using-properties-as-tags}]
+   ["65.3" {:rename-db-idents add-missing-db-ident-for-tags2 :fix add-missing-db-ident-for-tags}]
+   ["65.4" {:rename-db-idents fix-using-properties-as-tags2 :fix fix-using-properties-as-tags}]
    ["65.5" {:fix remove-block-order-for-tags}]
    ["65.6" {:fix update-extends-to-cardinality-many}]
    ["65.7" {:fix add-quick-add-page}]
    ["65.8" {:fix add-missing-page-name}]
-   ["65.9" {:properties [:logseq.property.embedding/hnsw-label-updated-at]}]
-   ["65.10" {:properties [:logseq.property/test1]}]
-   ["65.11" {:rename-db-idents [{:db-ident :logseq.property/test1
-                                 :new-db-ident :logseq.property/test2}]}]])
+   ["65.9" {:properties [:logseq.property.embedding/hnsw-label-updated-at]}]])
 
 (let [[major minor] (last (sort (map (comp (juxt :major :minor) db-schema/parse-schema-version first)
                                      schema-version->updates)))]
@@ -382,14 +415,6 @@
       (when (neg? compare-result)
         (js/console.warn (str "Current db schema-version is " db-schema/version ", max available schema-version is " max-schema-version))))))
 
-;;; some validations of schema-version->updates
-(doseq [[version migrate-updates] schema-version->updates]
-  (when (contains? (set (keys migrate-updates)) :fix)
-    (assert (= 1 (count migrate-updates))
-            (common-util/format
-             "migration(%s): :fix type cannot coexist with other types (:properties, :classes, :rename-db-idents)"
-             version))))
-
 (defn ensure-built-in-data-exists!
   "Return tx-data"
   [conn]
@@ -473,7 +498,8 @@
         new-class-idents (keep (fn [class]
                                  (when-let [db-ident (:db/ident class)]
                                    {:db/ident db-ident})) new-classes)
-        rename-db-idents-tx-data (rename-db-ident/rename-db-idents-migration-tx-data db rename-db-idents)
+        [rename-db-idents-tx-data rename-db-idents-coll]
+        (rename-db-ident/rename-db-idents-migration-tx-data db rename-db-idents)
         fixes (when (fn? fix)
                 (fix db))
         tx-data (if db-based?
@@ -482,18 +508,20 @@
         tx-data' (concat
                   [(sqlite-util/kv :logseq.kv/schema-version version)]
                   tx-data)
-        r (ldb/transact! conn tx-data' {:db-migrate? true})]
+        r (ldb/transact! conn tx-data' {:db-migrate? true})
+        migrate-updates (cond-> migrate-updates
+                          (seq rename-db-idents-coll) (assoc :rename-db-idents rename-db-idents-coll))]
     (println "DB schema migrated to" version)
     (assoc r :migrate-updates migrate-updates)))
 
 (defn migrate
   "Migrate 'frontend' datascript schema and data. To add a new migration,
   add an entry to schema-version->updates and bump db-schema/version"
-  [conn]
+  [conn & {:keys [target-version] :or {target-version db-schema/version}}]
   (when (ldb/db-based-graph? @conn)
     (let [db @conn
           version-in-db (db-schema/parse-schema-version (or (:kv/value (d/entity db :logseq.kv/schema-version)) 0))
-          compare-result (db-schema/compare-schema-version db-schema/version version-in-db)]
+          compare-result (db-schema/compare-schema-version target-version version-in-db)]
       (cond
         (zero? compare-result)
         nil
@@ -507,7 +535,7 @@
                 updates (keep (fn [[v updates]]
                                 (let [v* (db-schema/parse-schema-version v)]
                                   (when (and (neg? (db-schema/compare-schema-version version-in-db v*))
-                                             (not (pos? (db-schema/compare-schema-version v* db-schema/version))))
+                                             (not (pos? (db-schema/compare-schema-version v* target-version))))
                                     [v updates])))
                               schema-version->updates)
                 result-ks [:tx-data :db-before :db-after :migrate-updates]
@@ -519,9 +547,9 @@
             (swap! *upgrade-result-coll conj
                    (select-keys (ensure-built-in-data-exists! conn) result-ks))
             {:from-version version-in-db
-             :to-version db-schema/version
+             :to-version target-version
              :upgrade-result-coll @*upgrade-result-coll})
           (catch :default e
-            (prn :error (str "DB migration failed to migrate to " db-schema/version " from " version-in-db ":"))
+            (prn :error (str "DB migration failed to migrate to " target-version " from " version-in-db ":"))
             (js/console.error e)
             (throw e)))))))

src/main/frontend/worker/db/rename_db_ident.cljs

@@ -3,21 +3,26 @@
   (:require [datascript.core :as d]))
 
 (defn rename-db-idents-migration-tx-data
-  "Rename :db/ident and replace all usages as well. return tx-data.
-  rename-db-idents: coll of {:db-ident ..., :new-db-ident ...}
+  "Rename :db/ident and replace all usages as well.
+  rename-db-idents: fn to generate coll of {:db-ident ..., :new-db-ident ...}
   NOTE: this fn should only care about :db/ident changing, don't touch other attr/values"
   [db rename-db-idents]
-  (assert (every?
-           (fn [{:keys [db-ident new-db-ident]}]
-             (and (keyword? db-ident) (keyword? new-db-ident)))
-           rename-db-idents)
-          rename-db-idents)
-  (->> (for [{:keys [db-ident new-db-ident]} rename-db-idents
-             :let [ent (d/entity db db-ident)]
-             :when (some? ent)]
-         (cons {:db/id (:db/id ent) :db/ident new-db-ident}
-               (->> (d/q '[:find ?b ?v :in $ ?a :where [?b ?a ?v]] db db-ident)
-                    (mapcat (fn [[id v]]
-                              [[:db/retract id db-ident]
-                               [:db/add id new-db-ident v]])))))
-       (apply concat)))
+  (assert (fn? rename-db-idents))
+  (let [rename-db-idents-coll (rename-db-idents db)
+        *rename-db-idents-coll (atom [])
+        tx-data
+        (->> (for [{:keys [db-ident-or-block-uuid new-db-ident] :as rename-db-ident} rename-db-idents-coll
+                   :let [ent (d/entity db (if (keyword? db-ident-or-block-uuid)
+                                            db-ident-or-block-uuid
+                                            [:block/uuid db-ident-or-block-uuid]))
+                         old-db-ident (:db/ident ent)]]
+               (do (when (some? ent) (swap! *rename-db-idents-coll conj rename-db-ident))
+                   (cons {:db/id (:db/id ent) :db/ident new-db-ident}
+                         (some->> old-db-ident
+                                  (d/q '[:find ?b ?v :in $ ?a :where [?b ?a ?v]] db)
+                                  (mapcat (fn [[id v]]
+                                            [[:db/retract id old-db-ident]
+                                             [:db/add id new-db-ident v]]))))))
+             (apply concat)
+             doall)]
+    [tx-data @*rename-db-idents-coll]))

src/main/frontend/worker/rtc/client.cljs

@@ -289,7 +289,7 @@
   [rename-db-ident-ops-map]
   (keep (fn [[op-type op]]
           (when (keyword-identical? :rename-db-ident op-type)
-            [:rename-db-ident (select-keys (last op) [:db-ident :new-db-ident])]))
+            [:rename-db-ident (select-keys (last op) [:db-ident-or-block-uuid :new-db-ident])]))
         rename-db-ident-ops-map))
 
 (defn- gen-rename-db-ident-remote-ops

src/main/frontend/worker/rtc/client_op.cljs

@@ -25,7 +25,7 @@
      [:op :keyword]
      [:t :int]
      [:value [:map
-              [:db-ident :keyword]
+              [:db-ident-or-block-uuid [:or :keyword :uuid]]
               [:new-db-ident :keyword]]]]]
    [:move
     [:catn
@@ -87,6 +87,7 @@
   and move it to its own namespace."
   {:block/uuid {:db/unique :db.unique/identity}
    :db-ident {:db/unique :db.unique/identity}
+   :db-ident-or-block-uuid {:db/unique :db.unique/identity}
    :local-tx {:db/index true}
    :graph-uuid {:db/index true}
    :aes-key-jwk {:db/index true}
@@ -249,19 +250,19 @@
   [ops]
   (let [op-type :rename-db-ident
         sorted-ops (sort-by second ops)
-        db-ident->op
+        db-ident-or-block-uuid->op
         (reduce
          (fn [r op]
            (let [[_op-type _t value] op
-                 db-ident (:db-ident value)]
-             (assoc r db-ident op)))
+                 db-ident-or-block-uuid (:db-ident-or-block-uuid value)]
+             (assoc r db-ident-or-block-uuid op)))
          {} sorted-ops)]
     (mapcat
-     (fn [[db-ident op]]
-       (let [tmpid (str db-ident "-rename-db-ident")]
-         [[:db/add tmpid :db-ident db-ident]
+     (fn [[db-ident-or-block-uuid op]]
+       (let [tmpid (str db-ident-or-block-uuid "-rename-db-ident")]
+         [[:db/add tmpid :db-ident-or-block-uuid db-ident-or-block-uuid]
           [:db/add tmpid op-type op]]))
-     db-ident->op)))
+     db-ident-or-block-uuid->op)))
 
 (defn- partition-ops
   "Return [:update-kv-value-ops :rename-db-ident-ops block-ops]"
@@ -328,18 +329,18 @@
 
 (defn- get-all-rename-db-ident-ops*
   [db]
-  (let [db-ident-datoms (d/datoms db :avet :db-ident)
-        es (map :e db-ident-datoms)]
+  (let [db-ident-or-block-uuid-datoms (d/datoms db :avet :db-ident-or-block-uuid)
+        es (map :e db-ident-or-block-uuid-datoms)]
     (->> (map (fn [e] [e (d/datoms db :eavt e)]) es)
          (keep (fn [[e datoms]]
                  (let [op-map (into {}
                                     (keep (fn [datom]
                                             (let [a (:a datom)]
-                                              (when (or (keyword-identical? :db-ident a)
+                                              (when (or (keyword-identical? :db-ident-or-block-uuid a)
                                                         (contains? db-ident-rename-op-types a))
                                                 [a (:v datom)]))))
                                     datoms)]
-                   (when (and (:db-ident op-map) (> (count op-map) 1))
+                   (when (and (:db-ident-or-block-uuid op-map) (> (count op-map) 1))
                      [e op-map]))))
          (into {}))))
 

src/main/frontend/worker/rtc/gen_client_op.cljs

@@ -171,11 +171,12 @@
 
 (defn generate-rtc-rename-db-ident-ops
   [rename-db-idents]
-  (assert (every? (fn [{:keys [db-ident new-db-ident]}]
-                    (and (keyword? db-ident) (keyword? new-db-ident)))
+  (assert (every? (fn [{:keys [db-ident-or-block-uuid new-db-ident]}]
+                    (and (or (keyword? db-ident-or-block-uuid) (uuid? db-ident-or-block-uuid))
+                         (keyword? new-db-ident)))
                   rename-db-idents)
           rename-db-idents)
   (map
-   (fn [{:keys [db-ident new-db-ident]}]
-     [:rename-db-ident 0 {:db-ident db-ident :new-db-ident new-db-ident}])
+   (fn [{:keys [db-ident-or-block-uuid new-db-ident]}]
+     [:rename-db-ident 0 {:db-ident-or-block-uuid db-ident-or-block-uuid :new-db-ident new-db-ident}])
    rename-db-idents))

src/main/frontend/worker/rtc/malli_schema.cljs

@@ -33,7 +33,7 @@
    [:rename-db-ident
     [:cat :keyword
      [:map
-      [:db-ident :keyword]
+      [:db-ident-or-block-uuid [:or :keyword :uuid]]
       [:new-db-ident :keyword]]]]
    [:move
     [:cat :keyword

src/main/frontend/worker/rtc/migrate.cljs

@@ -4,35 +4,40 @@
             [datascript.core :as d]
             [frontend.worker.rtc.gen-client-op :as gen-client-op]))
 
+(def apply-conj (partial apply conj))
+
 (defn migration-results=>client-ops
   [{:keys [_from-version to-version upgrade-result-coll] :as _migration-result}]
-  (let [client-ops
-        (mapcat
-         (fn [{:keys [tx-data db-before db-after migrate-updates]}]
-           (cond
-             (:fix migrate-updates)
-             (let [{:keys [same-entity-datoms-coll id->same-entity-datoms]}
-                   (gen-client-op/group-datoms-by-entity tx-data)
-                   e->a->add?->v->t
-                   (update-vals
-                    id->same-entity-datoms
-                    gen-client-op/entity-datoms=>a->add?->v->t)]
-               (gen-client-op/generate-rtc-ops db-before db-after same-entity-datoms-coll e->a->add?->v->t))
-
-             (empty? (set/difference (set (keys migrate-updates)) #{:properties :classes :rename-db-idents}))
-             (let [property-ks (:properties migrate-updates)
-                   class-ks (:classes migrate-updates)
-                   rename-db-idents (:rename-db-idents migrate-updates)
-                   d-entity-fn (partial d/entity db-after)
-                   new-property-entities (keep d-entity-fn property-ks)
-                   new-class-entities (keep d-entity-fn class-ks)]
-               (concat (gen-client-op/generate-rtc-ops-from-property-entities new-property-entities)
-                       (gen-client-op/generate-rtc-ops-from-class-entities new-class-entities)
-                       (gen-client-op/generate-rtc-rename-db-ident-ops rename-db-idents)))))
-         upgrade-result-coll)
-        max-t (apply max 0 (map second client-ops))]
-    (conj (vec client-ops)
-          [:update-kv-value
-           max-t
-           {:db-ident :logseq.kv/schema-version
-            :value to-version}])))
+  (when to-version
+    (let [client-ops
+          (mapcat
+           (fn [{:keys [tx-data db-before db-after migrate-updates]}]
+             (let [*tx-data (atom [])]
+               (when-let [rename-db-idents (:rename-db-idents migrate-updates)]
+                 (swap! *tx-data apply-conj (gen-client-op/generate-rtc-rename-db-ident-ops rename-db-idents)))
+               (when (:fix migrate-updates)
+                 (let [{:keys [same-entity-datoms-coll id->same-entity-datoms]}
+                       (gen-client-op/group-datoms-by-entity tx-data)
+                       e->a->add?->v->t
+                       (update-vals
+                        id->same-entity-datoms
+                        gen-client-op/entity-datoms=>a->add?->v->t)]
+                   (swap! *tx-data apply-conj
+                          (gen-client-op/generate-rtc-ops
+                           db-before db-after same-entity-datoms-coll e->a->add?->v->t))))
+               (let [property-ks (seq (:properties migrate-updates))
+                     class-ks (:classes migrate-updates)
+                     d-entity-fn (partial d/entity db-after)
+                     new-property-entities (keep d-entity-fn property-ks)
+                     new-class-entities (keep d-entity-fn class-ks)]
+                 (swap! *tx-data apply-conj
+                        (concat (gen-client-op/generate-rtc-ops-from-property-entities new-property-entities)
+                                (gen-client-op/generate-rtc-ops-from-class-entities new-class-entities))))
+               @*tx-data))
+           upgrade-result-coll)
+          max-t (apply max 0 (map second client-ops))]
+      (conj (vec client-ops)
+            [:update-kv-value
+             max-t
+             {:db-ident :logseq.kv/schema-version
+              :value to-version}]))))

src/test/frontend/worker/rtc/migrate_test.cljs

@@ -1,16 +1,31 @@
 (ns frontend.worker.rtc.migrate-test
   (:require ["fs" :as fs-node]
             [cljs.pprint :as pp]
-            [cljs.test :refer [deftest]]
+            [cljs.test :refer [deftest is testing]]
+            [clojure.set :as set]
             [datascript.core :as d]
             [frontend.worker.db.migrate :as db-migrate]
             [frontend.worker.rtc.migrate :as rtc-migrate]
             [logseq.db :as ldb]))
 
 (deftest ^:focus migration-results=>client-ops
-  (let [db-transit (str (fs-node/readFileSync "src/test/migration/64.8.transit"))
-        db (ldb/read-transit-str db-transit)
-        conn (d/conn-from-db db)
-        migration-result (db-migrate/migrate conn)
-        client-ops (rtc-migrate/migration-results=>client-ops migration-result)]
-    (pp/pprint client-ops)))
+  (testing "65.2 => 65.3"
+    (let [db-transit (str (fs-node/readFileSync "src/test/migration/65.2.transit"))
+          db (ldb/read-transit-str db-transit)
+          conn (d/conn-from-db db)
+          migration-result (db-migrate/migrate conn {:target-version "65.3"})
+          client-ops (rtc-migrate/migration-results=>client-ops migration-result)]
+      (prn :migration-result "================================================================")
+      (pp/pprint (map (fn [r] [(:tx-data r) (select-keys (:migrate-updates r) [:rename-db-idents])])
+                      (:upgrade-result-coll migration-result)))
+      (prn :client-ops "================================================================")
+      (pp/pprint client-ops)
+      (testing "client-ops are generated correctly from migration-result"
+        (is (seq client-ops) "Client ops should not be empty")
+
+        (let [last-op (last client-ops)
+              schema-version-update? (= :update-kv-value (first last-op))]
+          (is schema-version-update? "The last op should be to update schema version")
+          (when schema-version-update?
+            (is (= :logseq.kv/schema-version (get-in last-op [2 :db-ident])) "The schema version key should be correct")
+            (is (= (:to-version migration-result) (get-in last-op [2 :value])) "The schema version should be updated to the new version")))))))