Add sync offline tests

deps/common/src/logseq/common/config.cljs

@@ -40,7 +40,6 @@
 (defonce views-page-name "$$$views")
 (defonce library-page-name "Library")
 (defonce quick-add-page-name "Quick add")
-(defonce recycle-page-name "Recycle")
 
 (defn local-relative-asset?
   [s]

deps/db-sync/src/logseq/db_sync/parent_missing.cljs

@@ -1,71 +0,0 @@
-(ns logseq.db-sync.parent-missing
-  (:require [datascript.core :as d]
-            [logseq.common.config :as common-config]
-            [logseq.db :as ldb]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.outliner.core :as outliner-core]
-            [logseq.outliner.transaction :as outliner-tx]))
-
-(defn ensure-recycle-page!
-  [conn]
-  (let [db @conn]
-    (or (ldb/get-built-in-page db common-config/recycle-page-name)
-        (let [page (-> (sqlite-util/build-new-page common-config/recycle-page-name)
-                       sqlite-create-graph/mark-block-as-built-in)
-              {:keys [db-after]} (ldb/transact! conn [page] {:db-sync/recycle-page? true
-                                                             :op :create-recycle-page})]
-          (d/entity db-after [:block/uuid (:block/uuid page)])))))
-
-(defn get-missing-parent-eids
-  [{:keys [db-after tx-data]}]
-  (->> tx-data
-       ;; block still exists while its parent has been gone
-       (filter (fn [d]
-                 (and (= :block/parent (:a d))
-                      (nil? (d/entity db-after (:v d)))
-                      (let [block (d/entity db-after (:e d))]
-                        (and (some? block)
-                             (nil? (:block/parent block))
-                             (not (ldb/page? block)))))))
-       (map :e)
-       distinct))
-
-(defn move-blocks-to-recycle!
-  [conn blocks]
-  (let [recycle-page (ensure-recycle-page! conn)]
-    (outliner-tx/transact!
-     {:op :fix-missing-parent
-      :transact-opts {:conn conn}}
-     (outliner-core/move-blocks! conn blocks recycle-page {:sibling? false}))))
-
-(defn fix-parent-missing!
-  [conn tx-report]
-  (when-let [missing-eids (seq (get-missing-parent-eids tx-report))]
-    (let [blocks (map (fn [eid]
-                        (d/entity (:db-after tx-report) eid))
-                      missing-eids)]
-      (move-blocks-to-recycle! conn blocks))))
-
-(defn- parent-missing?
-  [conn newly-ids [op _e a v]]
-  (and (= :block/parent a)
-       (= op :db/add)
-       (nil? (d/entity @conn v))
-       (not (contains? newly-ids (second v)))))
-
-(defn fix-parent-missing-for-tx-data!
-  [conn recycle-page-id tx-data]
-  (let [newly-ids (->> tx-data
-                       (keep (fn [[op _e a v]]
-                               (and (= :block/uuid a)
-                                    (= :db/add op)
-                                    v)))
-                       set)]
-    (->> tx-data
-         (map (fn [[op e a _v :as item]]
-                (if (parent-missing? conn newly-ids item)
-                  (do
-                    (prn :debug :item item :new-v recycle-page-id)
-                    [op e a recycle-page-id])
-                  item))))))

deps/db-sync/src/logseq/db_sync/worker.cljs

@@ -5,12 +5,9 @@
             [lambdaisland.glogi :as log]
             [lambdaisland.glogi.console :as glogi-console]
             [logseq.common.authorization :as authorization]
-            [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db-sync.common :as common :refer [cors-headers]]
             [logseq.db-sync.malli-schema :as db-sync-schema]
-            [logseq.db-sync.order :as sync-order]
-            [logseq.db-sync.parent-missing :as db-sync-parent-missing]
             [logseq.db-sync.protocol :as protocol]
             [logseq.db-sync.storage :as storage]
             [logseq.db.common.normalize :as db-normalize]
@@ -333,21 +330,9 @@
         conn (.-conn self)]
     (when-not conn
       (fail-fast :db-sync/missing-db {:op :apply-tx}))
-    (let [tx-data (protocol/transit->tx txs)]
-      (ldb/transact-with-temp-conn!
-       conn
-       {:apply-tx? true}
-       (fn [temp-conn *batch-tx-data]
-         (let [recycle-page-id (:db/id (db-sync-parent-missing/ensure-recycle-page! conn))
-               tx-data' (->> tx-data
-                             (db-sync-parent-missing/fix-parent-missing-for-tx-data! conn recycle-page-id)
-                             db-normalize/replace-attr-retract-with-retract-entity-v2)
-               tx-report (ldb/transact! temp-conn tx-data' {:op :apply-client-tx})]
-           (prn :debug :fix-parent-missing)
-           ;; TODO: fix cycle
-           (db-sync-parent-missing/fix-parent-missing! temp-conn tx-report)
-           (prn :debug :fix-duplicate-orders)
-           (sync-order/fix-duplicate-orders! temp-conn @*batch-tx-data))))
+    (let [tx-data (->> (protocol/transit->tx txs)
+                       db-normalize/replace-attr-retract-with-retract-entity-v2)]
+      (ldb/transact! conn tx-data {:op :apply-client-tx})
       (prn :debug :finished-db-transact)
       (let [new-t (storage/get-t sql)]
         ;; FIXME: no need to broadcast if client tx is less than remote tx

deps/db/src/logseq/db/sqlite/create_graph.cljs

@@ -130,7 +130,6 @@
 (def built-in-pages-names
   #{common-config/library-page-name
     common-config/quick-add-page-name
-    common-config/recycle-page-name
     "Contents"})
 
 (defn- validate-tx-for-duplicate-idents [tx]

deps/graph-parser/src/logseq/graph_parser/db.cljs

@@ -19,7 +19,7 @@
   (set/union
    (set built-in-markers)
    (set built-in-priorities)
-   #{"Favorites" "Contents" "card" common-config/recycle-page-name}))
+   #{"Favorites" "Contents" "card"}))
 
 (defn- page-title->block
   [title]

deps/outliner/src/logseq/outliner/core.cljs

@@ -857,10 +857,14 @@
                 ;; Replace entities with eid because Datascript doesn't support entity transaction
                 full-tx' (walk/prewalk
                           (fn [f]
-                            (if (de/entity? f)
+                            (cond
+                              (de/entity? f)
                               (if-let [id (id->new-uuid (:db/id f))]
                                 [:block/uuid id]
                                 (:db/id f))
+                              (map? f)
+                              (dissoc f :block/level)
+                              :else
                               f))
                           full-tx)]
             {:tx-data full-tx'

src/main/frontend/worker/db_sync.cljs

@@ -399,8 +399,6 @@
   (let [inflight @(:inflight client)
         local-tx (or (client-op/get-local-tx repo) 0)
         remote-tx (get @*repo->latest-remote-tx repo)]
-    (prn :debug :remote-tx remote-tx
-         :local-tx local-tx)
     (when (= local-tx remote-tx)        ; rebase
       (when (empty? inflight)
         (when-let [ws (:ws client)]
@@ -644,8 +642,7 @@
 (defn- rebase-apply-remote-tx! [repo client tx-data]
   (if-let [conn (worker-state/get-datascript-conn repo)]
     (try
-      (let [;; 1. revert local changes
-            local-txs (pending-txs repo)
+      (let [local-txs (pending-txs repo)
             reversed-tx-data (->> local-txs
                                   (mapcat :reversed-tx)
                                   reverse
@@ -682,7 +679,6 @@
                              (when (seq tx-data)
                                (let [rtc-tx-data (sanitize-remote-tx-data @temp-conn tx-data)
                                      tx-report (ldb/transact! temp-conn rtc-tx-data)]
-                                 (prn :debug :tx-data rtc-tx-data)
                                  (sync-order/fix-duplicate-orders! temp-conn (:tx-data tx-report))))))))]
 
         (when tx-report

src/test/frontend/worker/db_sync_test.cljs

@@ -2,80 +2,184 @@
   (:require [cljs.test :refer [deftest is testing run-test]]
             [datascript.core :as d]
             [frontend.worker.db-sync :as db-sync]
+            [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.state :as worker-state]
-            [logseq.common.config :as common-config]
-            [logseq.db :as ldb]
-            [logseq.db.test.helper :as db-test]))
+            [logseq.db.test.helper :as db-test]
+            [logseq.outliner.core :as outliner-core]))
 
 (def ^:private test-repo "test-db-sync-repo")
 
-(defn- with-datascript-conn
-  [conn f]
-  (let [prev @worker-state/*datascript-conns]
-    (reset! worker-state/*datascript-conns {test-repo conn})
+(defn- with-datascript-conns
+  [db-conn ops-conn f]
+  (let [db-prev @worker-state/*datascript-conns
+        ops-prev @worker-state/*client-ops-conns]
+    (reset! worker-state/*datascript-conns {test-repo db-conn})
+    (reset! worker-state/*client-ops-conns {test-repo ops-conn})
+    (when ops-conn
+      (d/listen! db-conn ::listen-db
+                 (fn [tx-report]
+                   (db-sync/enqueue-local-tx! test-repo tx-report))))
     (try
       (f)
       (finally
-        (reset! worker-state/*datascript-conns prev)))))
+        (reset! worker-state/*datascript-conns db-prev)
+        (reset! worker-state/*client-ops-conns ops-prev)))))
 
 (defn- setup-parent-child
   []
   (let [conn (db-test/create-conn-with-blocks
               {:pages-and-blocks
-               [{:page {:block/title "page1"}
+               [{:page {:block/title "page 1"}
                  :blocks [{:block/title "parent"
-                           :build/children [{:block/title "child"}]}]}]})
+                           :build/children [{:block/title "child 1"}
+                                            {:block/title "child 2"}
+                                            {:block/title "child 3"}]}]}]})
+        client-ops-conn (d/create-conn client-op/schema-in-db)
         parent (db-test/find-block-by-content @conn "parent")
-        child (db-test/find-block-by-content @conn "child")]
+        child1 (db-test/find-block-by-content @conn "child 1")
+        child2 (db-test/find-block-by-content @conn "child 2")
+        child3 (db-test/find-block-by-content @conn "child 3")]
     {:conn conn
+     :client-ops-conn client-ops-conn
      :parent parent
-     :child child}))
-
-(deftest create-recycle-page-when-missing-test
-  (testing "recycle page is created when missing during db-sync repair"
-    (let [{:keys [conn parent child]} (setup-parent-child)
-          recycle-page (ldb/get-built-in-page @conn common-config/recycle-page-name)]
-      (ldb/transact! conn [[:db/retractEntity (:db/id recycle-page)]])
-      (with-datascript-conn conn
-        (fn []
-          (#'db-sync/rebase-apply-remote-tx!
-           test-repo
-           nil
-           [[:db/retractEntity (:db/id parent)]])
-          (let [recycle-page' (ldb/get-built-in-page @conn common-config/recycle-page-name)]
-            (is (= common-config/recycle-page-name (:block/title recycle-page')))
-            (is (= "Recycle" (:block/title (:block/parent (d/entity @conn (:db/id child))))))))))))
+     :child1 child1
+     :child2 child2
+     :child3 child3}))
 
 (deftest reparent-block-when-cycle-detected-test
   (testing "cycle from remote sync reparent block to page root"
-    (let [{:keys [conn parent child]} (setup-parent-child)]
-      (with-datascript-conn conn
+    (let [{:keys [conn parent child1]} (setup-parent-child)]
+      (with-datascript-conns conn nil
         (fn []
           (#'db-sync/rebase-apply-remote-tx!
            test-repo
            nil
-           [[:db/add (:db/id parent) :block/parent (:db/id child)]])
+           [[:db/add (:db/id parent) :block/parent (:db/id child1)]])
           (let [parent' (d/entity @conn (:db/id parent))
-                child' (d/entity @conn (:db/id child))
+                child' (d/entity @conn (:db/id child1))
                 page' (:block/page parent')]
             (is (some? page'))
             (is (= (:db/id page') (:db/id (:block/parent parent'))))
             (is (= (:db/id parent') (:db/id (:block/parent child'))))))))))
 
-(deftest drop-missing-parent-update-test
-  (testing "drop invalid parent updates during remote rebase"
-    (let [{:keys [conn child]} (setup-parent-child)
-          child-uuid (:block/uuid child)
-          original-parent-uuid (:block/uuid (:block/parent (d/entity @conn (:db/id child))))
-          missing-parent-uuid (random-uuid)]
-      (prn :debug :missing-parent-uuid missing-parent-uuid)
-      (with-datascript-conn conn
+(deftest two-children-cycle-test
+  (testing "cycle from remote sync reparent block to page root"
+    (let [{:keys [conn client-ops-conn child1 child2]} (setup-parent-child)]
+      (with-datascript-conns conn client-ops-conn
         (fn []
+          (d/transact! conn [[:db/add (:db/id child1) :block/parent (:db/id child2)]])
           (#'db-sync/rebase-apply-remote-tx!
            test-repo
            nil
-           [[:db/add [:block/uuid child-uuid]
-             :block/parent [:block/uuid missing-parent-uuid]]])
-          (let [child' (d/entity @conn (:db/id child))]
-            (is (= original-parent-uuid
-                   (:block/uuid (:block/parent child'))))))))))
+           [[:db/add (:db/id child2) :block/parent (:db/id child1)]])
+          (let [child' (d/entity @conn (:db/id child1))
+                child2' (d/entity @conn (:db/id child2))]
+            (is (= "child 2" (:block/title (:block/parent child'))))
+            (is (= "page 1" (:block/title (:block/parent child2'))))))))))
+
+(deftest three-children-cycle-test
+  (testing "cycle from remote sync reparent block to page root"
+    (let [{:keys [conn client-ops-conn child1 child2 child3]} (setup-parent-child)]
+      (with-datascript-conns conn client-ops-conn
+        (fn []
+          (d/transact! conn [[:db/add (:db/id child2) :block/parent (:db/id child1)]
+                             [:db/add (:db/id child3) :block/parent (:db/id child2)]])
+          (#'db-sync/rebase-apply-remote-tx!
+           test-repo
+           nil
+           [[:db/add (:db/id child2) :block/parent (:db/id child3)]
+            [:db/add (:db/id child1) :block/parent (:db/id child2)]])
+          (let [child' (d/entity @conn (:db/id child1))
+                child2' (d/entity @conn (:db/id child2))
+                child3' (d/entity @conn (:db/id child3))]
+            (is (= "page 1" (:block/title (:block/parent child'))))
+            (is (= "page 1" (:block/title (:block/parent child2'))))
+            (is (= "child 2" (:block/title (:block/parent child3'))))))))))
+
+(deftest ignore-missing-parent-update-after-local-delete-test
+  (testing "remote parent retracted while local adds another child"
+    (let [{:keys [conn client-ops-conn parent child1]} (setup-parent-child)
+          child-uuid (:block/uuid child1)]
+      (with-datascript-conns conn client-ops-conn
+        (fn []
+          (outliner-core/insert-blocks! conn [{:block/title "child 4"}] parent {:sibling? false})
+          (#'db-sync/rebase-apply-remote-tx!
+           test-repo
+           nil
+           [[:db/retractEntity [:block/uuid (:block/uuid parent)]]])
+          (let [child' (d/entity @conn [:block/uuid child-uuid])]
+            (is (nil? child'))))))))
+
+(deftest fix-duplicate-orders-after-rebase-test
+  (testing "duplicate order updates are fixed after remote rebase"
+    (let [{:keys [conn client-ops-conn child1 child2]} (setup-parent-child)
+          order (:block/order (d/entity @conn (:db/id child1)))]
+      (with-datascript-conns conn client-ops-conn
+        (fn []
+          (d/transact! conn [[:db/add (:db/id child1) :block/title "child 1 local"]])
+          (#'db-sync/rebase-apply-remote-tx!
+           test-repo
+           nil
+           [[:db/add (:db/id child1) :block/order order]
+            [:db/add (:db/id child2) :block/order order]])
+          (let [child1' (d/entity @conn (:db/id child1))
+                child2' (d/entity @conn (:db/id child2))
+                orders [(:block/order child1') (:block/order child2')]]
+            (is (every? some? orders))
+            (is (= 2 (count (distinct orders))))))))))
+
+(deftest fix-duplicate-order-against-existing-sibling-test
+  (testing "duplicate order update is fixed when it collides with an existing sibling"
+    (let [{:keys [conn client-ops-conn child1 child2]} (setup-parent-child)
+          child2-order (:block/order (d/entity @conn (:db/id child2)))]
+      (with-datascript-conns conn client-ops-conn
+        (fn []
+          (d/transact! conn [[:db/add (:db/id child1) :block/title "child 1 local"]])
+          (#'db-sync/rebase-apply-remote-tx!
+           test-repo
+           nil
+           [[:db/add (:db/id child1) :block/order child2-order]])
+          (let [child1' (d/entity @conn (:db/id child1))
+                child2' (d/entity @conn (:db/id child2))]
+            (is (some? (:block/order child1')))
+            (is (not= (:block/order child1') (:block/order child2')))))))))
+
+(deftest fix-duplicate-orders-with-local-and-remote-new-blocks-test
+  (testing "local and remote new sibling blocks at the same location get unique orders"
+    (let [{:keys [conn client-ops-conn parent]} (setup-parent-child)
+          parent-id (:db/id parent)
+          page-uuid (:block/uuid (:block/page parent))
+          remote-uuid-1 (random-uuid)
+          remote-uuid-2 (random-uuid)]
+      (with-datascript-conns conn client-ops-conn
+        (fn []
+          (outliner-core/insert-blocks! conn [{:block/title "local 1"
+                                               :block/uuid (random-uuid)}
+                                              {:block/title "local 2"
+                                               :block/uuid (random-uuid)}]
+                                        parent
+                                        {:sibling? true})
+          (let [local1 (db-test/find-block-by-content @conn "local 1")
+                local2 (db-test/find-block-by-content @conn "local 2")]
+            (#'db-sync/rebase-apply-remote-tx!
+             test-repo
+             nil
+             [[:db/add -1 :block/uuid remote-uuid-1]
+              [:db/add -1 :block/title "remote 1"]
+              [:db/add -1 :block/parent [:block/uuid page-uuid]]
+              [:db/add -1 :block/page [:block/uuid page-uuid]]
+              [:db/add -1 :block/order (:block/order local1)]
+              [:db/add -1 :block/updated-at 1768308019312]
+              [:db/add -1 :block/created-at 1768308019312]
+              [:db/add -2 :block/uuid remote-uuid-2]
+              [:db/add -2 :block/title "remote 2"]
+              [:db/add -2 :block/parent [:block/uuid page-uuid]]
+              [:db/add -2 :block/page [:block/uuid page-uuid]]
+              [:db/add -2 :block/order (:block/order local2)]
+              [:db/add -2 :block/updated-at 1768308019312]
+              [:db/add -2 :block/created-at 1768308019312]]))
+          (let [parent' (d/entity @conn parent-id)
+                children (vec (:block/_parent parent'))
+                orders (map :block/order children)]
+            (is (every? some? orders))
+            (is (= (count orders) (count (distinct orders))))))))))