refactor: remove src/rtc_e2e_test

rtc-e2e-test already moved to clj-e2e

.carve/config.edn

@@ -1,6 +1,6 @@
 {;; Only lint production namespaces as most dev
  ;; namespaces are unused
- :paths ["src/main" "src/electron" "src/test" "src/rtc_e2e_test"]
+ :paths ["src/main" "src/electron" "src/test"]
  :api-namespaces [;; Ignore b/c too many false positives
                   frontend.db
                   ;; Used for debugging

bb.edn

@@ -164,9 +164,6 @@
   dev:lint-and-test
   logseq.tasks.dev/lint-and-test
 
-  dev:rtc-e2e-test
-  logseq.tasks.dev/rtc-e2e-test
-
   dev:gen-malli-kondo-config
   logseq.tasks.dev/gen-malli-kondo-config
 

deps.edn

@@ -68,10 +68,6 @@
                                 cider/cider-nrepl                {:mvn/version "0.55.1"}}
                   :main-opts   ["-m" "shadow.cljs.devtools.cli"]}
 
-           :rtc-e2e-test {:extra-paths ["src/rtc_e2e_test"]
-                          :extra-deps {cider/cider-nrepl                {:mvn/version "0.50.2"}}
-                          :main-opts ["-m" "shadow.cljs.devtools.cli"]}
-
            :bench {:extra-paths ["src/bench/"]
                    :extra-deps {olical/cljs-test-runner {:mvn/version "3.8.0"}
                                 fipp/fipp {:mvn/version "0.6.26"}}

karma.conf.js

@@ -1,19 +0,0 @@
-module.exports = function (config) {
-    config.set({
-        browsers: ['Chrome'],
-        // The directory where the output file lives
-        basePath: 'static/rtc-e2e-test',
-        // The file itself
-        files: ['main.js'],
-        frameworks: ['cljs-test'],
-        plugins: ['karma-cljs-test', 'karma-chrome-launcher'],
-        colors: true,
-        logLevel: config.LOG_INFO,
-        client: {
-            args: ["shadow.test.karma.init"],
-            singleRun: true,
-	    testvar: config.testvar,
-	    seed: config.seed
-        }
-    })
-};

scripts/src/logseq/tasks/dev.clj

@@ -26,17 +26,6 @@
   (dev-lint/dev)
   (test "-e" "long" "-e" "fix-me"))
 
-(defn rtc-e2e-test
-  "Run karma rtc-e2e-test"
-  [& [skip-compile?]]
-  (let [seed (hash (rand))
-        r0 (when-not skip-compile? (shell (str "clj -M:rtc-e2e-test compile rtc-e2e-test")))
-        c1 (async/go (shell (str "npx karma start --testvar=client1 --single-run --seed=" seed)))
-        c2 (async/go (shell (str "npx karma start --testvar=client2 --single-run --seed=" seed)))]
-    (when (and r0 (not= 0 (:exit r0)))
-      (throw (ex-info "compile failed" {:r r0})))
-    (prn :exit-code :client1 (:exit (async/<!! c1)) :client2 (:exit (async/<!! c2)))))
-
 (defn gen-malli-kondo-config
   "Generate clj-kondo type-mismatch config from malli schema
   .clj-kondo/metosin/malli-types/config.edn"

shadow-cljs.edn

@@ -6,8 +6,7 @@
 
  ;; "." for /static
  :dev-http {3001 ["static" "."]
-            3002 "static/mobile"
-            8021 "static/rtc-e2e-test"}
+            3002 "static/mobile"}
 
  :js-options {:js-package-dirs ["node_modules" "packages/tldraw/apps"]}
 
@@ -174,14 +173,6 @@
          :compiler-options {:static-fns false}
          :main frontend.test.frontend-node-test-runner/main}
 
-  :rtc-e2e-test {:target :karma
-                 :closure-defines {frontend.worker.rtc.const/RTC-E2E-TEST* true}
-                 :output-to "static/rtc-e2e-test/main.js"
-                 :devtools {:enabled true}
-                 :compiler-options {:source-map true
-                                    :warnings {:fn-deprecated false
-                                               :redef false}}}
-
   :gen-malli-kondo-config {:target :node-script
                            :closure-defines {frontend.util/NODETEST true}
                            :devtools {:enabled false}

src/main/frontend/common/missionary.cljs

@@ -131,11 +131,12 @@
   (let [cancel (task (or succ (constantly nil)) (or fail fail-case-default-handler))]
     #(cancel)))
 
-(defn run-task-throw
-  "Return the canceler"
-  [key' task & {:keys [succ]}]
-  (let [cancel (task (or succ #(log/info :key key' :succ %)) #(throw (ex-info "task stopped" {:key key' :e %})))]
-    #(cancel)))
+(comment
+  (defn run-task-throw
+    "Return the canceler"
+    [key' task & {:keys [succ]}]
+    (let [cancel (task (or succ #(log/info :key key' :succ %)) #(throw (ex-info "task stopped" {:key key' :e %})))]
+      #(cancel))))
 
 (defonce ^:private *background-task-cancelers ; key -> canceler
   (volatile! {}))

src/main/frontend/worker/rtc/const.cljs

@@ -4,9 +4,6 @@
             [logseq.db.frontend.kv-entity :as kv-entity]
             [logseq.db.frontend.property :as db-property]))
 
-(goog-define RTC-E2E-TEST* false)
-(def RTC-E2E-TEST RTC-E2E-TEST*)
-
 (defkeywords
   :rtc/ignore-attr-when-init-upload
   {:doc "keyword option for RTC. ignore this *attr* when initial uploading graph. Default false"}

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -8,7 +8,6 @@
             [frontend.common.thread-api :as thread-api]
             [frontend.worker-common.util :as worker-util]
             [frontend.worker.crypt :as crypt]
-            [frontend.worker.db-listener :as db-listener]
             [frontend.worker.db-metadata :as worker-db-metadata]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.const :as rtc-const]
@@ -20,8 +19,6 @@
             [lambdaisland.glogi :as log]
             [logseq.db :as ldb]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
-            [logseq.db.frontend.schema :as db-schema]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.outliner.pipeline :as outliner-pipeline]
             [malli.core :as ma]
@@ -161,8 +158,7 @@
             (client-op/remove-local-tx repo)
             (client-op/add-all-exists-asset-as-ops repo)
             (crypt/store-graph-keys-jwk repo aes-key-jwk)
-            (when-not rtc-const/RTC-E2E-TEST
-              (c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid}))))
+            (c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid})))
             (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-completed
                                                         :message "upload-graph completed"})
             {:graph-uuid graph-uuid})
@@ -269,29 +265,6 @@
        [schema-blocks (conj normal-blocks block)]))
    [[] []] blocks))
 
-(defn- create-graph-for-rtc-test
-  "TODO: remove this fn
-  it's complex to setup db-worker related stuff, when I only want to test rtc related logic"
-  [repo init-tx-data other-tx-data]
-  (let [conn (d/create-conn db-schema/schema)
-        db-initial-data (sqlite-create-graph/build-db-initial-data "")]
-    (swap! worker-state/*datascript-conns assoc repo conn)
-    (d/transact! conn db-initial-data {:initial-db? true
-                                       :frontend.worker.pipeline/skip-store-conn rtc-const/RTC-E2E-TEST})
-    (db-listener/listen-db-changes! repo conn)
-    (d/transact! conn init-tx-data {:rtc-download-graph? true
-                                    :gen-undo-ops? false
-                                    ;; only transact db schema, skip validation to avoid warning
-                                    :frontend.worker.pipeline/skip-validate-db? true
-                                    :frontend.worker.pipeline/skip-store-conn rtc-const/RTC-E2E-TEST
-                                    :persist-op? false})
-    (d/transact! conn other-tx-data {:rtc-download-graph? true
-                                     :gen-undo-ops? false
-                                     :frontend.worker.pipeline/skip-store-conn rtc-const/RTC-E2E-TEST
-                                     :persist-op? false})
-    (transact-remote-schema-version! repo)
-    (<transact-block-refs! repo nil)))
-
 (defn- blocks-resolve-temp-id
   [schema-blocks blocks]
   (let [uuids (map :block/uuid blocks)
@@ -388,37 +361,35 @@
       (client-op/update-local-tx repo remote-t)
       (rtc-log-and-state/update-local-t graph-uuid remote-t)
       (rtc-log-and-state/update-remote-t graph-uuid remote-t)
-      (if rtc-const/RTC-E2E-TEST
-        (create-graph-for-rtc-test repo init-tx-data tx-data)
-        (c.m/<?
-         (p/do!
-          ((@thread-api/*thread-apis :thread-api/create-or-open-db) repo {:close-other-db? false})
-          ((@thread-api/*thread-apis :thread-api/export-db) repo)
-          (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transact-graph-data-to-db-1
-                                                        :message (str "transacting init data(" (count init-tx-data) ")")
-                                                        :graph-uuid graph-uuid})
-          ((@thread-api/*thread-apis :thread-api/transact)
-           repo init-tx-data
-           {:rtc-download-graph? true
-            :gen-undo-ops? false
+      (c.m/<?
+       (p/do!
+        ((@thread-api/*thread-apis :thread-api/create-or-open-db) repo {:close-other-db? false})
+        ((@thread-api/*thread-apis :thread-api/export-db) repo)
+        (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transact-graph-data-to-db-1
+                                                      :message (str "transacting init data(" (count init-tx-data) ")")
+                                                      :graph-uuid graph-uuid})
+        ((@thread-api/*thread-apis :thread-api/transact)
+         repo init-tx-data
+         {:rtc-download-graph? true
+          :gen-undo-ops? false
             ;; only transact db schema, skip validation to avoid warning
-            :frontend.worker.pipeline/skip-validate-db? true
-            :persist-op? false}
-           (worker-state/get-context))
-          (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transact-graph-data-to-db-2
-                                                        :message (str "transacting other data(" (count tx-data) ")")
-                                                        :graph-uuid graph-uuid})
-          (p/doseq [tx-data* (partition-all 500 tx-data)]
-            ((@thread-api/*thread-apis :thread-api/transact)
-             repo tx-data* {:rtc-download-graph? true
-                            :gen-undo-ops? false
-                            :persist-op? false} (worker-state/get-context))
-            (p/delay 10))
-          (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transact-graph-data-to-db-3
-                                                        :message "transacting remote schema version"
-                                                        :graph-uuid graph-uuid})
-          (transact-remote-schema-version! repo)
-          (<transact-block-refs! repo graph-uuid))))
+          :frontend.worker.pipeline/skip-validate-db? true
+          :persist-op? false}
+         (worker-state/get-context))
+        (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transact-graph-data-to-db-2
+                                                      :message (str "transacting other data(" (count tx-data) ")")
+                                                      :graph-uuid graph-uuid})
+        (p/doseq [tx-data* (partition-all 500 tx-data)]
+          ((@thread-api/*thread-apis :thread-api/transact)
+           repo tx-data* {:rtc-download-graph? true
+                          :gen-undo-ops? false
+                          :persist-op? false} (worker-state/get-context))
+          (p/delay 10))
+        (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :transact-graph-data-to-db-3
+                                                      :message "transacting remote schema version"
+                                                      :graph-uuid graph-uuid})
+        (transact-remote-schema-version! repo)
+        (<transact-block-refs! repo graph-uuid)))
       (shared-service/broadcast-to-clients! :add-repo {:repo repo}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;
@@ -490,8 +461,7 @@
                                                             :message "transacted all blocks"
                                                             :graph-uuid graph-uuid})
               (client-op/update-graph-uuid repo graph-uuid)
-              (when-not rtc-const/RTC-E2E-TEST
-                (c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid}))))
+              (c.m/<? (worker-db-metadata/<store repo (pr-str {:kv/value graph-uuid})))
               (worker-state/set-rtc-downloading-graph! false)
               (rtc-log-and-state/rtc-log :rtc.log/download {:sub-type :download-completed
                                                             :message "download completed"
@@ -600,7 +570,4 @@
            datoms)))]
     (prn ::count (count refs-tx))
     ;; (prn ::take-20 (take 20 (sort-by second > (into [] (frequencies (map last refs-tx))))))
-    )
-
-
-  )
+    ))

src/main/frontend/worker/rtc/log_and_state.cljs

@@ -9,10 +9,6 @@
 
 (def ^:private *rtc-log (atom nil))
 
-(def rtc-log-flow
-  "used by rtc-e2e-test"
-  (m/watch *rtc-log))
-
 (def ^:private rtc-log-type-schema
   (vec
    (concat

src/main/frontend/worker/rtc/remote_update.cljs

@@ -618,8 +618,7 @@ so need to pull earlier remote-data from websocket."})
           (js/console.groupCollapsed "rtc/apply-remote-ops-log")
           (batch-tx/with-batch-tx-mode conn {:rtc-tx? true
                                              :persist-op? false
-                                             :gen-undo-ops? false
-                                             :frontend.worker.pipeline/skip-store-conn rtc-const/RTC-E2E-TEST}
+                                             :gen-undo-ops? false}
             (worker-util/profile :ensure-refed-blocks-exist (ensure-refed-blocks-exist repo conn refed-blocks))
             (worker-util/profile :apply-remote-update-page-ops (apply-remote-update-page-ops repo conn update-page-ops))
             (worker-util/profile :apply-remote-move-ops (apply-remote-move-ops repo conn sorted-move-ops))

src/rtc_e2e_test/basic_edits_test.cljs

@@ -1,28 +0,0 @@
-(ns basic-edits-test
-  (:require [client-steps]
-            [cljs.test :as t :refer [deftest]]
-            [const]
-            [fixture]
-            [helper]
-            [missionary.core :as m]))
-
-(t/use-fixtures :once
-  fixture/install-some-consts
-  fixture/install-example-db-fixture
-  fixture/clear-test-remote-graphs-fixture
-  fixture/upload-example-graph-fixture
-  fixture/build-conn-by-download-example-graph-fixture)
-
-(deftest basic-edits-test
-  (t/async
-   done
-   (js/Promise.
-    (if const/is-client1?
-      (m/sp
-        (doseq [task client-steps/client1-steps]
-          (m/? task))
-        (done))
-      (m/sp
-        (doseq [task client-steps/client2-steps]
-          (m/? task))
-        (done))))))

src/rtc_e2e_test/client_steps.cljs

@@ -1,264 +0,0 @@
-(ns client-steps
-  (:require [cljs.test :as t :refer [is]]
-            [const]
-            [datascript.core :as d]
-            [frontend.worker.rtc.client-op :as client-op]
-            [frontend.worker.rtc.core :as rtc-core]
-            [helper]
-            [frontend.common.missionary :as c.m]
-            [logseq.db :as ldb]
-            [missionary.core :as m]))
-
-(def ^:private step0
-  {:client1
-   (m/sp
-     (let [conn (helper/get-downloaded-test-conn)
-           tx-data (const/tx-data-map :create-page)]
-       (helper/transact! conn tx-data)
-       (is (=
-            #{[:update-page const/page1-uuid]
-              [:update const/page1-uuid
-               [[:block/title "[\"~#'\",\"basic-edits-test\"]" true]
-                [:block/created-at "[\"~#'\",1724836490809]" true]
-                [:block/updated-at "[\"~#'\",1724836490809]" true]
-                [:block/type "[\"~#'\",\"page\"]" true]]]
-              [:move const/block1-uuid]
-              [:update const/block1-uuid
-               [[:block/updated-at "[\"~#'\",1724836490810]" true]
-                [:block/created-at "[\"~#'\",1724836490810]" true]
-                [:block/title "[\"~#'\",\"block1\"]" true]]]}
-            (set (map helper/simplify-client-op (client-op/get-all-block-ops const/downloaded-test-repo)))))))
-   :client2 nil})
-
-(def ^:private step1
-  "client1: start rtc, wait page1, client1->remote
-  client2: start rtc, wait page1, remote->client2"
-  {:client1
-   (m/sp
-     (let [r (m/? (rtc-core/new-task--rtc-start false))]
-       (is (nil? r))
-       (m/? (helper/new-task--wait-all-client-ops-sent))))
-   :client2
-   (m/sp
-     (let [r (m/? (rtc-core/new-task--rtc-start false))]
-       (is (nil? r)))
-     (m/?
-      (c.m/backoff
-       {}
-       (m/sp
-         (let [conn (helper/get-downloaded-test-conn)
-               page1 (d/pull @conn '[*] [:block/uuid const/page1-uuid])
-               block1 (d/pull @conn '[*] [:block/uuid const/block1-uuid])]
-           (when-not (:block/uuid page1)
-             (throw (ex-info "wait page1 synced" {:missionary/retry true})))
-           (is
-            (= {:block/title "basic-edits-test"
-                :block/name "basic-edits-test"
-                :block/type "page"}
-               (select-keys page1 [:block/title :block/name :block/type])))
-           (is
-            (= {:block/title "block1"
-                :block/order "a0"
-                :block/parent {:db/id (:db/id page1)}}
-               (select-keys block1 [:block/title :block/order :block/parent]))))))))})
-
-(def ^:private step2
-  "client1: insert 500 blocks, wait for changes to sync to remote
-  client2: wait for blocks to sync from remote"
-  {:client1
-   (m/sp
-     (let [conn (helper/get-downloaded-test-conn)]
-       (helper/transact! conn (const/tx-data-map :insert-500-blocks))
-       (m/? (helper/new-task--wait-all-client-ops-sent))))
-   :client2
-   (c.m/backoff
-    {}
-    (m/sp
-      (let [conn (helper/get-downloaded-test-conn)
-            page (d/pull @conn '[*] [:block/uuid const/page2-uuid])]
-        (when-not (:block/uuid page)
-          (throw (ex-info "wait page to be synced" {:missionary/retry true})))
-        (let [blocks (ldb/sort-by-order (ldb/get-page-blocks @conn (:db/id page)))]
-          (is (= 500 (count blocks)))
-          (is (= (map #(str "x" %) (range 500))
-                 (map :block/title blocks)))))))})
-
-(def ^:private step3
-  "client1:
-  1. add #task properties to block1 (`const/block1-uuid`)
-  2. wait to be synced
-  3. toggle block1 status to TODO
-  4. wait to be synced
-  5. toggle block1 status to DOING
-  6. wait to be synced
-  client2:
-  1. wait the block&its properties to be synced"
-  {:client1
-   (m/sp
-     (let [conn (helper/get-downloaded-test-conn)
-           tx-data1 (const/tx-data-map :step3-add-task-properties-to-block1)
-           tx-data2 (const/tx-data-map :step3-toggle-status-TODO)
-           tx-data3 (const/tx-data-map :step3-toggle-status-DOING)]
-       (helper/transact! conn tx-data1)
-       (m/? (helper/new-task--wait-all-client-ops-sent))
-       (helper/transact! conn tx-data2)
-       (m/? (helper/new-task--wait-all-client-ops-sent))
-       (helper/transact! conn tx-data3)
-       (m/? (helper/new-task--wait-all-client-ops-sent))))
-   :client2
-   (c.m/backoff
-    {}
-    (m/sp
-      (let [conn (helper/get-downloaded-test-conn)
-            block1 (d/pull @conn
-                           [{:block/tags [:db/ident]}
-                            {:logseq.property/status [:db/ident]}
-                            {:logseq.property/deadline [:block/journal-day]}]
-                           [:block/uuid const/block1-uuid])]
-        (when-not (= :logseq.property/status.doing (:db/ident (:logseq.property/status block1)))
-          (throw (ex-info "wait block1's task properties to be synced" {:missionary/retry true})))
-        (is (= {:block/tags [{:db/ident :logseq.class/Task}],
-                :logseq.property/status {:db/ident :logseq.property/status.doing}
-                :logseq.property/deadline {:block/journal-day 20240907}}
-               block1)))))})
-(def ^:private step4
-  "client1:
-
-  client2:
-"
-  {:client1
-   (m/sp nil)
-   :client2
-   (m/sp nil)})
-
-(def ^:private step5
-  "client1:
-  - insert some blocks in page2
-  - wait to be synced
-  - wait a signal from client2
-  - send a signal to client2
-  - stop rtc
-  - move some blocks
-  - start rtc
-  - wait to be synced
-  - wait client2's message, which contains the result of client2's block tree,
-    and compare them with blocks in client1
-  client2:
-  - wait inserted blocks synced
-  - send a signal to client1
-  - wait a signal from client1
-  - stop rtc
-  - move some blocks
-  - start rtc
-  - wait to be synced
-  - send a message to client1 contains client2's block tree to client1"
-  {:client1
-   (m/sp
-     (let [conn (helper/get-downloaded-test-conn)
-           tx-data1 (const/tx-data-map :move-blocks-concurrently-1)
-           tx-data2 (const/tx-data-map :move-blocks-concurrently-client1)]
-       (helper/transact! conn tx-data1)
-       (m/? (helper/new-task--wait-all-client-ops-sent))
-       (m/? (helper/new-task--client1-sync-barrier-2->1 "move-blocks-concurrently-signal"))
-       (m/? helper/new-task--stop-rtc)
-       (helper/transact! conn tx-data2)
-       (is (nil? (m/? (rtc-core/new-task--rtc-start false))))
-       (m/? (helper/new-task--wait-all-client-ops-sent))
-       (m/? (helper/new-task--client1-sync-barrier-2->1 "step5"))
-       (let [message (m/? (helper/new-task--wait-message-from-other-client
-                           (fn [message] (= "move-blocks-concurrently-page-blocks" (:id message)))
-                           :retry-message "move-blocks-concurrently-page-blocks"))
-             client2-page-blocks (:page-blocks message)
-             client1-page-blocks (ldb/get-page-blocks @conn (:db/id (d/entity @conn [:block/uuid const/page3-uuid]))
-                                                      :pull-keys '[:block/uuid :block/title :block/order
-                                                                   {:block/parent [:block/uuid]}])]
-         (is (= (set client1-page-blocks) (set client2-page-blocks))))))
-   :client2
-   (m/sp
-     (let [conn (helper/get-downloaded-test-conn)]
-       (m/?
-        (c.m/backoff
-         (take 4 c.m/delays)
-         (m/sp
-           (let [page3 (d/pull @conn '[*] [:block/uuid const/page3-uuid])
-                 page3-blocks (some->> (:db/id page3)
-                                       (ldb/get-page-blocks @conn))]
-             (when-not (:block/uuid page3)
-               (throw (ex-info "wait page3 synced" {:missionary/retry true})))
-             (is (= 6 (count page3-blocks)))))))
-       (m/? (helper/new-task--client2-sync-barrier-2->1 "move-blocks-concurrently-signal"))
-       (m/? helper/new-task--stop-rtc)
-       (helper/transact! conn (const/tx-data-map :move-blocks-concurrently-client2))
-       (is (nil? (m/? (rtc-core/new-task--rtc-start false))))
-       (m/? (helper/new-task--wait-all-client-ops-sent))
-       (m/? (helper/new-task--client2-sync-barrier-2->1 "step5"))
-       (m/? (helper/new-task--send-message-to-other-client
-             {:id "move-blocks-concurrently-page-blocks"
-              :page-blocks (ldb/get-page-blocks @conn (:db/id (d/entity @conn [:block/uuid const/page3-uuid]))
-                                                :pull-keys '[:block/uuid :block/title :block/order
-                                                             {:block/parent [:block/uuid]}])}))))})
-
-(def ^:private step6
-  "Delete blocks test-1
-client1:
-- insert some blocks
-- wait to be synced
-- stop rtc
-- delete blocks
-- start rtc
-- wait to be synced
-
-client2:
-- wait blocks from client1
-- wait delete-blocks changes synced from client1
-- check block-tree"
-  {:client1
-   (m/sp
-     (let [conn (helper/get-downloaded-test-conn)
-           tx-data1 (const/tx-data-map :step6-delete-blocks-client1-1)
-           tx-data2 (const/tx-data-map :step6-delete-blocks-client1-2)]
-       (helper/transact! conn tx-data1)
-       (m/? (helper/new-task--wait-all-client-ops-sent))
-       (m/? (helper/new-task--client1-sync-barrier-1->2 "step6"))
-       (m/? helper/new-task--stop-rtc)
-       (helper/transact! conn tx-data2)
-       (let [r (m/? (rtc-core/new-task--rtc-start false))]
-         (is (nil? r))
-         (m/? (helper/new-task--wait-all-client-ops-sent)))))
-   :client2
-   (m/sp
-     (let [conn (helper/get-downloaded-test-conn)]
-       (m/? (helper/new-task--client2-sync-barrier-1->2 "step6"))
-       (m/?
-        (c.m/backoff
-         {}
-         (m/sp
-           (let [page (d/pull @conn '[*] [:block/uuid const/step6-page-uuid])
-                 page-blocks (when-let [page-id (:db/id page)]
-                               (ldb/get-page-blocks @conn page-id
-                                                    :pull-keys '[:block/uuid {:block/parent [:block/uuid]}]))]
-             (when-not (= 1 (count page-blocks))
-               (throw (ex-info "wait delete-blocks changes synced"
-                               {:missionary/retry true
-                                :page-blocks page-blocks})))
-             (is (= {:block/uuid const/step6-block3-uuid
-                     :block/parent {:block/uuid const/step6-page-uuid}}
-                    (select-keys (first page-blocks) [:block/uuid :block/parent])))))))))})
-
-(defn- wrap-print-step-info
-  [steps client]
-  (map-indexed
-   (fn [idx step]
-     (m/sp
-       (helper/log "start step" idx)
-       (some-> (get step client) m/?)
-       (helper/log "end step" idx)))
-   steps))
-
-(def ^:private all-steps [step0 step1 step2 step3 step4 step5 step6])
-
-(def client1-steps
-  (wrap-print-step-info all-steps :client1))
-
-(def client2-steps
-  (wrap-print-step-info all-steps :client2))

src/rtc_e2e_test/const.cljs

@@ -1,229 +0,0 @@
-(ns const
-  "Consts for rtc e2e tests"
-  (:require [logseq.db.common.order :as db-order]))
-
-(assert (exists? js/__karma__))
-(def seed js/__karma__.config.seed)
-(def testvar js/__karma__.config.testvar)
-(prn :karma-config :seed seed :testvar testvar)
-
-(def is-client1? (= "client1" testvar))
-
-(def test-token "TEST-TOKEN")
-(def test-graph-name (str "TEST-REPO-" seed))
-(def test-repo (str "logseq_db_TEST-REPO-" seed))
-
-(def downloaded-test-graph-name "TEST-REPO-downloaded")
-(def downloaded-test-repo "logseq_db_TEST-REPO-downloaded")
-
-;;; tests data
-(def message-page-uuid #uuid "a3da426a-4202-4a79-8e97-13f4862b0270")
-
-(def page1-uuid #uuid "c051d36f-98b3-4afb-b52a-d5a06bd8591d")
-(def page2-uuid #uuid "91d3e320-d2a6-47ae-96a7-8a366ab96cbb")
-(def page3-uuid #uuid "9a846640-2b63-4298-9ad6-8ca6c1285016")
-
-(def block1-uuid #uuid "aa6d5e60-5d3a-4468-812f-bd60dc9639fb")
-
-;;; ----- move-blocks-concurrently case -----
-(def block2-uuid #uuid "a78e19fc-7e9a-4f61-8988-0e9a649bc875")
-(def block3-uuid #uuid "226166d8-1380-4d7a-9fe1-f98e2d583259")
-(def block4-uuid #uuid "fb8f05d2-9d91-492e-81e2-8a0b65f09d8c")
-(def block5-uuid #uuid "f3c48e62-1726-4492-b42a-a36f4de7b32f")
-(def block6-uuid #uuid "23f51a53-db85-465a-9f18-6ca94e59f56c")
-(def block7-uuid #uuid "83f99937-fe0a-4d33-81ce-7fe5837baad3")
-;;; ----- delete-blocks case ---------
-(def step6-page-uuid #uuid "e22dafa5-b3b4-405d-b93d-470caa420e10")
-(def step6-block1-uuid #uuid "776acd4a-d011-4985-bfc2-14ee7bbd6a28")
-(def step6-block2-uuid #uuid "ba3998c2-8059-4f9e-9e76-2760d2f14512")
-(def step6-block3-uuid #uuid "f9ce5393-370a-43dd-a721-aaa5ef83d3ff")
-(def step6-block4-uuid #uuid "db00bb0d-2bef-49e7-96ed-b4882cdf5686")
-(def step6-block5-uuid #uuid "d34e8a9c-5e87-4511-b982-2bf2ebc82607")
-
-(def ^:large-vars/data-var tx-data-map
-  {:create-page
-   [{:db/id "page"
-     :block/name "basic-edits-test"
-     :block/title "basic-edits-test"
-     :block/uuid page1-uuid
-     :block/created-at 1724836490809
-     :block/updated-at 1724836490809
-     :block/type "page"}
-    {:block/uuid block1-uuid
-     :block/updated-at 1724836490810
-     :block/created-at 1724836490810
-     :block/title "block1"
-     :block/parent "page"
-     :block/order "a0"
-     :block/page "page"}]
-   :insert-500-blocks
-   (cons {:db/id "page"
-          :block/uuid page2-uuid
-          :block/name "insert-500-blocks"
-          :block/title "insert-500-blocks"
-          :block/created-at 1725024677501
-          :block/updated-at 1725024677501
-          :block/type "page"}
-         (map (fn [i order]
-                {:block/uuid (random-uuid)
-                 :block/created-at 1725024677501
-                 :block/updated-at 1725024677501
-                 :block/title (str "x" i)
-                 :block/parent "page"
-                 :block/order order
-                 :block/page "page"})
-              (range 500) (db-order/gen-n-keys 500 "a0" "a1")))
-   :step3-add-task-properties-to-block1
-   [{:db/id "id-0907"
-     :block/uuid #uuid "00000001-2024-0907-0000-000000000000"
-     :block/updated-at 1725455235108
-     :block/created-at 1725455235108
-     :block/journal-day 20240907
-     :block/title "Sep 7th, 2024"
-     :block/name "sep 7th, 2024"
-     :block/type "journal"}
-    {:block/uuid block1-uuid
-     :block/updated-at 1725454876718
-     :block/tags :logseq.class/Task
-     :logseq.property/status :logseq.property/status.done
-     :logseq.property/deadline "id-0907"}]
-   :step3-toggle-status-TODO
-   [{:block/uuid block1-uuid
-     :logseq.property/status :logseq.property/status.todo}]
-   :step3-toggle-status-DOING
-   [{:block/uuid block1-uuid
-     :logseq.property/status :logseq.property/status.doing}]
-   :move-blocks-concurrently-1
-   [{:db/id "page"
-     :block/uuid page3-uuid
-     :block/name "move-blocks-concurrently"
-     :block/title "move-blocks-concurrently"
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/type "page"}
-    {:block/uuid block2-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x1"
-     :block/parent "page"
-     :block/order "a0"
-     :block/page "page"}
-    {:block/uuid block3-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x2"
-     :block/parent "page"
-     :block/order "a1"
-     :block/page "page"}
-    {:block/uuid block4-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x3"
-     :block/parent "page"
-     :block/order "a2"
-     :block/page "page"}
-    {:block/uuid block5-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x4"
-     :block/parent "page"
-     :block/order "a3"
-     :block/page "page"}
-    {:block/uuid block6-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x5"
-     :block/parent "page"
-     :block/order "a4"
-     :block/page "page"}
-    {:block/uuid block7-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x6"
-     :block/parent "page"
-     :block/order "a5"
-     :block/page "page"}]
-   :move-blocks-concurrently-client1
-   [{:block/uuid block6-uuid
-     :block/parent [:block/uuid block3-uuid]
-     :block/order "a0"}
-    {:block/uuid block5-uuid
-     :block/parent [:block/uuid block6-uuid]
-     :block/order "a0"}
-    {:block/uuid block4-uuid
-     :block/parent [:block/uuid block2-uuid]
-     :block/order "a0"}
-    {:block/uuid block7-uuid
-     :block/parent [:block/uuid page3-uuid]
-     :block/order (db-order/gen-key "a0" "a1")}]
-   :move-blocks-concurrently-client2
-   [{:block/uuid block2-uuid
-     :block/order "a2V"}
-    {:block/uuid block5-uuid
-     :block/parent [:block/uuid block3-uuid]
-     :block/order "a0"}
-    {:block/uuid block6-uuid
-     :block/parent [:block/uuid block5-uuid]
-     :block/order "a0"}
-    {:block/uuid block4-uuid
-     :block/parent [:block/uuid block7-uuid]
-     :block/order "a0"}]
-   :step6-delete-blocks-client1-1
-   ;; - 1
-   ;;   - 2
-   ;; - 3
-   ;;   - 4
-   ;;     - 5
-   [{:db/id "page"
-     :block/uuid step6-page-uuid
-     :block/name "step6-delete-blocks"
-     :block/title "step6-delete-blocks"
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/type "page"}
-    {:db/id "b1"
-     :block/uuid step6-block1-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x1"
-     :block/parent "page"
-     :block/order "a0"
-     :block/page "page"}
-    {:db/id "b2"
-     :block/uuid step6-block2-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x2"
-     :block/parent "b1"
-     :block/order "a0"
-     :block/page "page"}
-    {:db/id "b3"
-     :block/uuid step6-block3-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x3"
-     :block/parent "page"
-     :block/order "a1"
-     :block/page "page"}
-    {:db/id "b4"
-     :block/uuid step6-block4-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x4"
-     :block/parent "b3"
-     :block/order "a0"
-     :block/page "page"}
-    {:db/id "b5"
-     :block/uuid step6-block5-uuid
-     :block/created-at 1725024677501
-     :block/updated-at 1725024677501
-     :block/title "x5"
-     :block/parent "b4"
-     :block/order "a0"
-     :block/page "page"}]
-   :step6-delete-blocks-client1-2
-   ;; only block3 left
-   [[:db/retractEntity [:block/uuid step6-block1-uuid]]
-    [:db/retractEntity [:block/uuid step6-block2-uuid]]
-    [:db/retractEntity [:block/uuid step6-block4-uuid]]
-    [:db/retractEntity [:block/uuid step6-block5-uuid]]]})

src/rtc_e2e_test/fixture.cljs

@@ -1,82 +0,0 @@
-(ns fixture
-  (:require [cljs.test :as t]
-            [const]
-            [datascript.core :as d]
-            [example]
-            [frontend.common.missionary :as c.m]
-            [frontend.worker.rtc.client-op :as client-op]
-            [frontend.worker.rtc.db-listener]
-            [frontend.worker.state :as worker-state]
-            [helper]
-            [missionary.core :as m]))
-
-(def graph-schema-version "0")
-
-(defn- transact-graph-schema-version
-  [conn]
-  (d/transact! conn [{:db/ident :logseq.kv/schema-version
-                      :kv/value graph-schema-version}]))
-
-(def install-some-consts
-  {:before
-   (fn []
-     (reset! worker-state/*rtc-ws-url "wss://ws-dev.logseq.com/rtc-sync?token=%s"))})
-
-(def install-example-db-fixture
-  {:before
-   (fn []
-     (prn :test-repo const/test-repo)
-     (swap! worker-state/*client-ops-conns assoc const/test-repo (d/create-conn client-op/schema-in-db))
-     (let [conn (d/conn-from-db example/example-db)]
-       (transact-graph-schema-version conn)
-       (swap! worker-state/*datascript-conns assoc const/test-repo conn)))
-   :after
-   (fn []
-     (swap! worker-state/*datascript-conns dissoc const/test-repo)
-     (swap! worker-state/*client-ops-conns dissoc const/test-repo))})
-
-(def clear-test-remote-graphs-fixture
-  {:before
-   #(when const/is-client1?
-      (t/async
-       done
-       (c.m/run-task-throw
-        :clear-test-remote-graphs
-        (m/sp
-          (m/? helper/new-task--clear-all-test-remote-graphs)
-          (done)))))})
-
-(def upload-example-graph-fixture
-  {:before
-   #(when const/is-client1?
-      (t/async
-       done
-       (c.m/run-task-throw
-        :upload-example-graph-fixture
-        (m/sp
-          (swap! worker-state/*datascript-conns dissoc const/downloaded-test-repo)
-          (swap! worker-state/*client-ops-conns assoc
-                 const/downloaded-test-repo (d/create-conn client-op/schema-in-db))
-          (let [{:keys [graph-uuid]} (m/? helper/new-task--upload-example-graph)]
-            (assert (some? graph-uuid))
-            (m/? (helper/new-task--wait-creating-graph graph-uuid))
-            (println :uploaded-graph graph-uuid))
-          (done)))))})
-
-(def build-conn-by-download-example-graph-fixture
-  {:before
-   #(t/async
-     done
-     (c.m/run-task-throw
-      :build-conn-by-download-example-graph-fixture
-      (m/sp
-        (swap! worker-state/*datascript-conns dissoc const/downloaded-test-repo)
-        (swap! worker-state/*client-ops-conns assoc
-               const/downloaded-test-repo (d/create-conn client-op/schema-in-db))
-        (let [graph-uuid (m/? helper/new-task--get-remote-example-graph-uuid)]
-          (assert (some? graph-uuid))
-          (m/? (helper/new-task--download-graph graph-uuid const/downloaded-test-graph-name)))
-        (done))))
-   :after
-   #(do (swap! worker-state/*datascript-conns dissoc const/downloaded-test-repo)
-        (swap! worker-state/*client-ops-conns dissoc const/downloaded-test-repo))})

src/rtc_e2e_test/helper.cljs

@@ -1,208 +0,0 @@
-(ns helper
-  (:require [cljs.test :as t :refer [is]]
-            [const]
-            [datascript.core :as d]
-            [datascript.transit :as dt]
-            [fixture]
-            [frontend.common.missionary :as c.m]
-            [frontend.worker.rtc.client-op :as client-op]
-            [frontend.worker.rtc.core :as rtc.core]
-            [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
-            [frontend.worker.state :as worker-state]
-            [logseq.db :as ldb]
-            [logseq.db.common.order :as db-order]
-            [logseq.outliner.batch-tx :as batch-tx]
-            [meander.epsilon :as me]
-            [missionary.core :as m]))
-
-(defn log
-  [& objs]
-  (apply println (if const/is-client1? "[client1]" "[client2]") objs))
-
-(def new-task--upload-example-graph
-  (rtc.core/new-task--upload-graph const/test-token const/test-repo const/test-graph-name))
-
-(defn new-task--wait-creating-graph
-  [graph-uuid]
-  (c.m/backoff
-   {}
-   (m/sp
-     (let [graphs (m/? (rtc.core/new-task--get-graphs const/test-token))
-           graph (some (fn [graph] (when (= graph-uuid (:graph-uuid graph)) graph)) graphs)]
-       (when-not graph
-         (throw (ex-info "graph not exist" {:graph-uuid graph-uuid})))
-       (log "waiting for graph " graph-uuid " finish creating")
-       (when (= "creating" (:graph-status graph))
-         (throw (ex-info "wait creating-graph" {:missionary/retry true})))))))
-
-(def new-task--clear-all-test-remote-graphs
-  (m/sp
-    (let [graphs (m/? (rtc.core/new-task--get-graphs const/test-token))
-          test-graphs (filter (fn [graph]
-                                (not= "deleting" (:graph-status graph)))
-                              graphs)]
-      (doseq [graph test-graphs]
-        (m/? (rtc.core/new-task--delete-graph const/test-token (:graph-uuid graph) fixture/graph-schema-version))
-        (log :deleted-graph (:graph-name graph) (:graph-uuid graph))))))
-
-(def new-task--get-remote-example-graph-uuid
-  (c.m/backoff
-   {}
-   (m/sp
-     (let [graphs (m/? (rtc.core/new-task--get-graphs const/test-token))
-           graph
-           (some (fn [graph]
-                   (when (= const/test-graph-name (:graph-name graph))
-                     graph))
-                 graphs)]
-       (when (= "deleting" (:graph-status graph))
-         (throw (ex-info "example graph status is \"deleting\", check server's background-upload-graph log"
-                         {:graph-name (:graph-name graph)
-                          :graph-uuid (:graph-uuid graph)})))
-       (when-not graph
-         (throw (ex-info "wait remote-example-graph" {:missionary/retry true
-                                                      :graphs graphs})))
-       (when (= "creating" (:graph-status graph))
-         (throw (ex-info "wait remote-example-graph (creating)" {:missionary/retry true
-                                                                 :graphs graphs})))
-       (:graph-uuid graph)))))
-
-(defn new-task--download-graph
-  [graph-uuid graph-name]
-  (m/sp
-    (let [download-info-uuid (m/? (rtc.core/new-task--request-download-graph
-                                   const/test-token graph-uuid fixture/graph-schema-version))
-          result (m/? (rtc.core/new-task--wait-download-info-ready
-                       const/test-token download-info-uuid graph-uuid fixture/graph-schema-version 60000))
-          {:keys [_download-info-uuid
-                  download-info-s3-url
-                  _download-info-tx-instant
-                  _download-info-t
-                  _download-info-created-at]} result]
-      (when (= result :timeout)
-        (throw (ex-info "wait download-info-ready timeout" {})))
-      (m/? (rtc.core/new-task--download-graph-from-s3
-            graph-uuid graph-name download-info-s3-url)))))
-
-(defn get-downloaded-test-conn
-  []
-  {:post [(some? %)]}
-  (worker-state/get-datascript-conn const/downloaded-test-repo))
-
-(defn simplify-client-op
-  [client-op]
-  #_:clj-kondo/ignore
-  (me/find
-   client-op
-    [?op-type _ {:block-uuid ?block-uuid :av-coll [[!a !v _ !add] ...]}]
-    [?op-type ?block-uuid (map vector !a !v !add)]
-
-    [?op-type _ {:block-uuid ?block-uuid}]
-    [?op-type ?block-uuid]))
-
-(defn new-task--wait-all-client-ops-sent
-  [& {:keys [timeout] :or {timeout 10000}}]
-  (m/sp
-    (let [r (m/? (m/timeout
-                  (m/reduce (fn [_ v]
-                              (when (and (= :rtc.log/push-local-update (:type v))
-                                         (empty? (client-op/get-all-block-ops const/downloaded-test-repo)))
-                                (is (nil? (:ex-data v)))
-                                (reduced v)))
-                            rtc-log-and-state/rtc-log-flow)
-                  timeout :timeout))]
-      (is (not= :timeout r)))))
-
-(defn new-task--send-message-to-other-client
-  [message]
-  (m/sp
-    (let [conn (get-downloaded-test-conn)
-          message-page-id (:db/id (ldb/get-page @conn const/message-page-uuid))
-          sorted-blocks (when message-page-id
-                          (ldb/sort-by-order (ldb/get-page-blocks @conn message-page-id)))
-          min-order (db-order/gen-key nil (:block/order (first sorted-blocks)))
-          tx-data [{:db/id "page"
-                    :block/uuid const/message-page-uuid
-                    :block/name "message-page"
-                    :block/title "message-page"
-                    :block/created-at 1725024677501
-                    :block/updated-at 1725024677501
-                    :block/type "page"}
-                   {:block/uuid (random-uuid)
-                    :block/parent "page"
-                    :block/order min-order
-                    :block/title (dt/write-transit-str message)
-                    :block/page "page"
-                    :block/updated-at 1724836490810
-                    :block/created-at 1724836490810}]]
-      (batch-tx/with-batch-tx-mode conn {:e2e-test const/downloaded-test-repo :frontend.worker.pipeline/skip-store-conn true}
-        (d/transact! conn tx-data))
-      (m/? (new-task--wait-all-client-ops-sent))
-      (log :sent-message message))))
-
-(defn new-task--wait-message-from-other-client
-  "Return a task that return message from other client"
-  [block-title-pred-fn & {:keys [retry-message retry-count] :or {retry-count 4}}]
-  (c.m/backoff
-   {:delay-seq (take retry-count c.m/delays)}
-   (m/sp
-     (let [conn (get-downloaded-test-conn)
-           message-page-id (:db/id (ldb/get-page @conn const/message-page-uuid))
-           first-block (when message-page-id
-                         (first (ldb/sort-by-order (ldb/get-page-blocks @conn message-page-id))))
-           first-block-title (some->> (:block/title first-block) dt/read-transit-str)]
-       (when-not (and (some? first-block-title)
-                      (block-title-pred-fn first-block-title))
-         (throw (ex-info (str "wait message from other client " retry-message) {:missionary/retry true})))
-       first-block-title))))
-
-(defn new-task--client1-sync-barrier-1->2
-  [message]
-  (m/sp
-    (m/? (new-task--send-message-to-other-client (str message "-client1")))
-    (m/? (new-task--wait-message-from-other-client #(= (str message "-client2") %)))
-    (log "sync-barrier-1->2" message)))
-
-(defn new-task--client2-sync-barrier-1->2
-  [message]
-  (m/sp
-    (m/? (new-task--wait-message-from-other-client #(= (str message "-client1") %)))
-    (m/? (new-task--send-message-to-other-client (str message "-client2")))
-    (log "sync-barrier-1->2" message)))
-
-(defn new-task--client1-sync-barrier-2->1
-  [message]
-  (m/sp
-    (m/? (new-task--wait-message-from-other-client #(= (str message "-client2") %)))
-    (m/? (new-task--send-message-to-other-client (str message "-client1")))
-    (log "sync-barrier-2->1" message)))
-
-(defn new-task--client2-sync-barrier-2->1
-  [message]
-  (m/sp
-    (m/? (new-task--send-message-to-other-client (str message "-client2")))
-    (m/? (new-task--wait-message-from-other-client #(= (str message "-client1") %)))
-    (log "sync-barrier-2->1" message)))
-
-(defn transact!
-  [conn tx-data]
-  {:pre [(seq tx-data)]}
-  (batch-tx/with-batch-tx-mode conn {:e2e-test const/downloaded-test-repo :frontend.worker.pipeline/skip-store-conn true}
-    (d/transact! conn tx-data)))
-
-(def new-task--stop-rtc
-  (m/sp
-    (rtc.core/rtc-stop)
-    (let [r (m/?
-             (m/timeout
-              (m/reduce
-               (fn [_ v]
-                 (when (= :rtc.log/cancelled (:type v))
-                   (log :debug-stop-rtc v)
-                   (reduced v)))
-               rtc-log-and-state/rtc-log-flow)
-              3000
-              :timeout))]
-      (is (not= :timeout r))
-      ;; sleep 0.1s to ensure *rtc-lock released
-      (m/? (m/sleep 100)))))