test(rtc): update upload-download-graph-test

shadow-cljs.edn

@@ -93,7 +93,15 @@
                  :test-dir "static/rtc-e2e-test"
                  :closure-defines {frontend.worker.rtc.const/RTC-E2E-TEST true}
                  :devtools {:enabled false}
-                 :compiler-options {:static-fns false}}
+                 :compiler-options {:infer-externs      :auto
+                                    :output-feature-set :es-next-in
+                                    :source-map true
+                                    :externs ["datascript/externs.js"
+                                              "externs.js"]
+                                    :warnings {:fn-deprecated false
+                                               :redef false}
+                                    :cross-chunk-method-motion false
+                                    :static-fns false}}
 
   :gen-malli-kondo-config {:target :node-script
                            :closure-defines {frontend.util/NODETEST true}

src/main/frontend/common/missionary_util.cljs

@@ -60,6 +60,10 @@
   [task key & {:keys [succ fail]}]
   (task (or succ #(prn key :succ %)) (or fail #(js/console.log key %))))
 
+(defn run-task-throw
+  [task key & {:keys [succ]}]
+  (task (or succ #(prn key :succ %)) #(throw (ex-info "task failed" {:key key :e %}))))
+
 (comment
   (defn >!
     "Return a task that

src/main/frontend/worker/pipeline.cljs

@@ -65,6 +65,10 @@
               (assert (= (count (distinct (map :block/order children))) (count children))
                       (str ":block/order is not unique for children blocks, parent id: " (:db/id parent))))))))))
 
+(sr/defkeyword :skip-store-conn
+  "tx-meta option, skip `d/store` on conn.
+default = false")
+
 (defn invoke-hooks
   [repo conn {:keys [tx-meta] :as tx-report} context]
   (when-not (:pipeline-replace? tx-meta)
@@ -77,7 +81,9 @@
                           (when (seq path-refs)
                             (ldb/transact! conn path-refs {:pipeline-replace? true}))
                           (do
-                            (when-not (exists? js/process) (d/store @conn))
+                            (when-not (or (exists? js/process)
+                                          (:skip-store-conn tx-meta false))
+                              (d/store @conn))
                             tx-report))
               full-tx-data (concat (:tx-data tx-report) (:tx-data tx-report'))
               final-tx-report (assoc tx-report'
@@ -117,7 +123,9 @@
               tx-report' (if (seq replace-tx)
                            (ldb/transact! conn replace-tx {:pipeline-replace? true})
                            (do
-                             (when-not (exists? js/process) (d/store @conn))
+                             (when-not (or (exists? js/process)
+                                           (:skip-store-conn tx-meta false))
+                               (d/store @conn))
                              tx-report))
               _ (validate-db! repo conn tx-report tx-meta context)
               full-tx-data (concat (:tx-data tx-report)

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -5,6 +5,7 @@
             [clojure.set :as set]
             [datascript.core :as d]
             [frontend.common.missionary-util :as c.m]
+            [frontend.worker.db-listener :as db-listener]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
@@ -13,6 +14,8 @@
             [frontend.worker.util :as worker-util]
             [logseq.db :as ldb]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
+            [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.outliner.pipeline :as outliner-pipeline]
             [malli.core :as ma]
@@ -202,7 +205,8 @@
                          {:db/id (:db/id block)
                           :block/refs refs})))
                    datoms)]
-      (ldb/transact! conn refs-tx {:outliner-op :rtc-download-rebuild-block-refs}))))
+      (ldb/transact! conn refs-tx (cond-> {:outliner-op :rtc-download-rebuild-block-refs}
+                                    rtc-const/RTC-E2E-TEST (assoc :skip-store-conn true))))))
 
 (defn- block->schema-map
   [block]
@@ -226,14 +230,35 @@
        [schema-blocks (conj normal-blocks block)]))
    [[] []] blocks))
 
+
+(defn- create-graph-for-rtc-test
+  "it's complex to setup db-worker related stuff, when I only want to test rtc related logic"
+  [repo init-tx-data other-tx-data]
+  (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
+        db-initial-data (sqlite-create-graph/build-db-initial-data "{}")]
+    (swap! worker-state/*datascript-conns assoc repo conn)
+    (d/transact! conn db-initial-data {:initial-db? true :skip-store-conn rtc-const/RTC-E2E-TEST})
+    (db-listener/listen-db-changes! repo conn)
+    (d/transact! conn init-tx-data {:rtc-download-graph? true
+                                    :gen-undo-ops? false
+                                    ;; only transact db schema, skip validation to avoid warning
+                                    :skip-validate-db? true
+                                    :skip-store-conn rtc-const/RTC-E2E-TEST
+                                    :persist-op? false})
+    (d/transact! conn other-tx-data {:rtc-download-graph? true
+                                     :gen-undo-ops? false
+                                     :skip-store-conn rtc-const/RTC-E2E-TEST
+                                     :persist-op? false})
+    (transact-block-refs! repo)))
+
 (defn- new-task--transact-remote-all-blocks
   [all-blocks repo graph-uuid]
   (let [{:keys [t blocks]} all-blocks
         card-one-attrs (blocks->card-one-attrs blocks)
         blocks (worker-util/profile :convert-card-one-value-from-value-coll
-                                    (map (partial convert-card-one-value-from-value-coll card-one-attrs) blocks))
+                 (map (partial convert-card-one-value-from-value-coll card-one-attrs) blocks))
         blocks (worker-util/profile :normalize-remote-blocks
-                                    (normalized-remote-blocks-coercer blocks))
+                 (normalized-remote-blocks-coercer blocks))
         ;;TODO: remove this, client/schema already converted to :db/cardinality, :db/valueType by remote,
         ;; and :client/schema should be removed by remote too
         blocks (map #(dissoc % :client/schema) blocks)
@@ -249,20 +274,22 @@
       (client-op/update-local-tx repo t)
       (rtc-log-and-state/update-local-t graph-uuid t)
       (rtc-log-and-state/update-remote-t graph-uuid t)
-      (m/?
-       (c.m/await-promise
-        (p/do!
-         (.createOrOpenDB worker-obj repo (ldb/write-transit-str {:close-other-db? false}))
-         (.exportDB worker-obj repo)
-         (.transact worker-obj repo init-tx-data {:rtc-download-graph? true
-                                                  :gen-undo-ops? false
-                                                  ;; only transact db schema, skip validation to avoid warning
-                                                  :skip-validate-db? true
-                                                  :persist-op? false} (worker-state/get-context))
-         (.transact worker-obj repo tx-data {:rtc-download-graph? true
-                                             :gen-undo-ops? false
-                                             :persist-op? false} (worker-state/get-context))
-         (transact-block-refs! repo))))
+      (if rtc-const/RTC-E2E-TEST
+        (create-graph-for-rtc-test repo init-tx-data tx-data)
+        (m/?
+         (c.m/await-promise
+          (p/do!
+            (.createOrOpenDB worker-obj repo (ldb/write-transit-str {:close-other-db? false}))
+            (.exportDB worker-obj repo)
+            (.transact worker-obj repo init-tx-data {:rtc-download-graph? true
+                                                     :gen-undo-ops? false
+                                                     ;; only transact db schema, skip validation to avoid warning
+                                                     :skip-validate-db? true
+                                                     :persist-op? false} (worker-state/get-context))
+            (.transact worker-obj repo tx-data {:rtc-download-graph? true
+                                                :gen-undo-ops? false
+                                                :persist-op? false} (worker-state/get-context))
+            (transact-block-refs! repo)))))
       (worker-util/post-message :add-repo {:repo repo}))))
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;

src/rtc_e2e_test/const.cljs

@@ -2,4 +2,7 @@
   "Consts for rtc e2e tests")
 
 (def test-token "TEST-TOKEN")
-(def test-repo "TEST-REPO")
+(def test-graph-name "TEST-REPO")
+(def test-repo "logseq_db_TEST-REPO")
+(def downloaded-test-graph-name "TEST-REPO-downloaded")
+(def downloaded-test-repo "logseq_db_TEST-REPO-downloaded")

src/rtc_e2e_test/download_upload_test.cljs

@@ -2,23 +2,30 @@
   "RTC e2e tests for download-graph and upload-graph"
   (:require [cljs.test :as t :refer [deftest]]
             [const]
+            [datascript.core :as d]
             [example]
             [fixture]
             [frontend.common.missionary-util :as c.m]
             [helper]
-            [missionary.core :as m]))
+            [missionary.core :as m]
+            [frontend.worker.state :as worker-state]))
 
 (t/use-fixtures :once
   fixture/install-some-consts
   fixture/install-example-db-fixture
   fixture/clear-test-remote-graphs-fixture)
 
-(deftest upload-graph-test
+(deftest upload-download-graph-test
   (t/async
    done
-   (c.m/run-task
+   (c.m/run-task-throw
     (m/sp
+      (println :example-db-block-count (count (d/datoms @(helper/get-example-test-conn) :avet :block/uuid)))
       (let [{:keys [graph-uuid]} (m/? helper/new-task--upload-example-graph)]
-        (m/? (helper/new-task--wait-creating-graph graph-uuid)))
+        (m/? (helper/new-task--wait-creating-graph graph-uuid))
+        (m/? (helper/new-task--download-graph graph-uuid))
+        (let [conn (helper/get-downloaded-test-conn)]
+          (println :repos (keys @worker-state/*datascript-conns))
+          (println :block-count (count (d/datoms @conn :avet :block/uuid)))))
       (done))
-    :upload-graph-test)))
+    :upload-download-graph-test)))

src/rtc_e2e_test/helper.cljs

@@ -2,7 +2,8 @@
   (:require [const]
             [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.core :as rtc.core]
-            [missionary.core :as m]))
+            [missionary.core :as m]
+            [frontend.worker.state :as worker-state]))
 
 (def new-task--upload-example-graph
   (rtc.core/new-task--upload-graph const/test-token const/test-repo const/test-repo))
@@ -29,3 +30,26 @@
                               graphs)]
       (doseq [graph test-graphs]
         (m/? (rtc.core/new-task--delete-graph const/test-token (:graph-uuid graph)))))))
+
+(defn new-task--download-graph
+  [graph-uuid]
+  (m/sp
+    (let [download-info-uuid (m/? (rtc.core/new-task--request-download-graph const/test-token graph-uuid))
+          result (m/? (rtc.core/new-task--wait-download-info-ready const/test-token download-info-uuid graph-uuid 60000))
+          {:keys [_download-info-uuid
+                  download-info-s3-url
+                  _download-info-tx-instant
+                  _download-info-t
+                  _download-info-created-at]} result]
+      (when (= result :timeout)
+        (throw (ex-info "wait download-info-ready timeout" {})))
+      (m/? (rtc.core/new-task--download-graph-from-s3
+            graph-uuid const/downloaded-test-graph-name download-info-s3-url)))))
+
+(defn get-downloaded-test-conn
+  []
+  (worker-state/get-datascript-conn const/downloaded-test-repo))
+
+(defn get-example-test-conn
+  []
+  (worker-state/get-datascript-conn const/test-repo))