feat(rtc,wip): migration support in rtc

src/main/frontend/worker/db/migrate.cljs

@@ -4,6 +4,7 @@
             [clojure.walk :as walk]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
+            [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.util :as worker-util]
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
@@ -378,6 +379,7 @@
         (js/console.warn (str "Current db schema-version is " db-schema/version ", max available schema-version is " max-schema-version))))))
 
 (defn ensure-built-in-data-exists!
+  "Return tx-data"
   [conn]
   (let [*uuids (atom {})
         data (->> (sqlite-create-graph/build-db-initial-data "")
@@ -430,11 +432,13 @@
                    [:block/uuid (@*uuids (second f))]
                    :else
                    f))
-               data)]
-    (d/transact! conn data' {:fix-db? true
-                             :db-migrate? true})))
+               data)
+        r (d/transact! conn data' {:fix-db? true
+                                   :db-migrate? true})]
+    (:tx-data r)))
 
 (defn- upgrade-version!
+  "Return tx-data"
   [conn db-based? version {:keys [properties classes fix]}]
   (let [version (db-schema/parse-schema-version version)
         db @conn
@@ -462,14 +466,15 @@
         tx-data (if db-based? (concat new-class-idents new-properties new-classes fixes) fixes)
         tx-data' (concat
                   [(sqlite-util/kv :logseq.kv/schema-version version)]
-                  tx-data)]
-    (ldb/transact! conn tx-data' {:db-migrate? true})
-    (println "DB schema migrated to" version)))
+                  tx-data)
+        r (ldb/transact! conn tx-data' {:db-migrate? true})]
+    (println "DB schema migrated to" version)
+    (:tx-data r)))
 
 (defn migrate
   "Migrate 'frontend' datascript schema and data. To add a new migration,
   add an entry to schema-version->updates and bump db-schema/version"
-  [conn]
+  [repo conn]
   (when (ldb/db-based-graph? @conn)
     (let [db @conn
           version-in-db (db-schema/parse-schema-version (or (:kv/value (d/entity db :logseq.kv/schema-version)) 0))
@@ -489,11 +494,17 @@
                                   (when (and (neg? (db-schema/compare-schema-version version-in-db v*))
                                              (not (pos? (db-schema/compare-schema-version v* db-schema/version))))
                                     [v updates])))
-                              schema-version->updates)]
+                              schema-version->updates)
+                *tx-data-coll (atom [])]
             (println "DB schema migrated from" version-in-db)
             (doseq [[v m] updates]
-              (upgrade-version! conn db-based? v m))
-            (ensure-built-in-data-exists! conn))
+              (let [tx-data (upgrade-version! conn db-based? v m)]
+                (swap! *tx-data-coll conj tx-data)))
+            (client-op/add-migration-datoms! repo version-in-db db-schema/version @*tx-data-coll)
+            (swap! *tx-data-coll conj (ensure-built-in-data-exists! conn))
+            {:from-version version-in-db
+             :to-version db-schema/version
+             :tx-data-coll @*tx-data-coll})
           (catch :default e
             (prn :error (str "DB migration failed to migrate to " db-schema/version " from " version-in-db ":"))
             (js/console.error e)

src/main/frontend/worker/db_worker.cljs

@@ -297,7 +297,7 @@
 
         (gc-sqlite-dbs! db client-ops-db conn {})
 
-        (db-migrate/migrate conn)
+        (db-migrate/migrate repo conn)
 
         (db-listener/listen-db-changes! repo (get @*datascript-conns repo))))))
 

src/main/frontend/worker/rtc/client_op.cljs

@@ -82,6 +82,7 @@
    :local-tx {:db/index true}
    :graph-uuid {:db/index true}
    :aes-key-jwk {:db/index true}
+   :migration-datoms {:db/index true}
 
    ;; device
    :device/uuid {:db/unique :db.unique/identity}
@@ -469,3 +470,10 @@
       (m/ap
         (let [_ (m/?> (c.m/throttle 100 db-updated-flow))]
           (datom-count-fn @conn))))))
+
+(defn add-migration-datoms!
+  [repo from-version to-version datoms]
+  (when-let [conn (worker-state/get-client-ops-conn repo)]
+    (d/transact! conn [{:migration-datoms {:datoms datoms
+                                           :from from-version
+                                           :to to-version}}])))

src/main/frontend/worker/rtc/migrate.cljs

@@ -47,3 +47,24 @@
                            (migration-updates->client-ops db client-schema-version)))]
     (client-op/add-ops! repo ops)
     ops))
+
+(defn local-datoms-tx-data=>remote-tx-data
+  [db datoms-tx-data]
+  (let [e->datoms (group-by :e datoms-tx-data)
+        e->datomvec-coll
+        (update-vals
+         e->datoms
+         (fn [datoms]
+           (let [e (:e (first datoms))
+                 need-block-uuid-datom?
+                 (every?
+                  (fn [{:keys [a added]}]
+                    (or (not= :block/uuid a)
+                        (and (= :block/uuid a) (false? added))))
+                  datoms)
+                 block-uuid (when need-block-uuid-datom? (:block/uuid (d/entity db e)))
+                 datoms* (cond->> datoms
+                           (and need-block-uuid-datom? block-uuid)
+                           (cons (d/datom e :block/uuid block-uuid)))]
+             (map (fn [{:keys [e a v tx added]}] [e a v tx added]) datoms*))))]
+    e->datomvec-coll))

src/test/frontend/worker/rtc/migrate_test.cljs

@@ -0,0 +1,14 @@
+(ns frontend.worker.rtc.migrate-test
+  (:require ["fs" :as fs-node]
+            [cljs.test :refer [deftest is testing]]
+            [logseq.db :as ldb]
+            [frontend.worker.db.migrate :as db-migrate]
+            [datascript.core :as d]
+            [cljs.pprint :as pp]))
+
+(deftest ^:focus local-datoms-tx-data=>remote-tx-data-test
+  (let [db-transit (str (fs-node/readFileSync "src/test/migration/64.8.transit"))
+        db (ldb/read-transit-str db-transit)
+        conn (d/conn-from-db db)
+        tx-data-coll (db-migrate/migrate "rtc-migrate-test" conn)]
+    (pp/pprint tx-data-coll)))