enhance(rtc,e2ee): make it compatible with unencrypted graphs

deps/db/src/logseq/db.cljs

@@ -562,6 +562,10 @@
   [db]
   (when db (get-key-value db :logseq.kv/remote-schema-version)))
 
+(defn get-graph-rtc-e2ee?
+  [db]
+  (when db (get-key-value db :logseq.kv/graph-rtc-e2ee?)))
+
 (def get-all-properties db-db/get-all-properties)
 (def get-class-extends db-class/get-class-extends)
 (def get-classes-parents db-db/get-classes-parents)

deps/db/src/logseq/db/common/initial_data.cljs

@@ -348,6 +348,7 @@
                        [:logseq.kv/db-type
                         :logseq.kv/schema-version
                         :logseq.kv/graph-uuid
+                        :logseq.kv/graph-rtc-e2ee?
                         :logseq.kv/latest-code-lang
                         :logseq.kv/graph-backup-folder
                         :logseq.kv/graph-text-embedding-model-name

deps/db/src/logseq/db/frontend/kv_entity.cljs

@@ -36,4 +36,5 @@ RTC won't start when major-schema-versions don't match"
 
      :logseq.kv/graph-text-embedding-model-name   {:doc "Graph's text-embedding model name"
                                                    :rtc {:rtc/ignore-entity-when-init-upload true
-                                                         :rtc/ignore-entity-when-init-download true}})))
+                                                         :rtc/ignore-entity-when-init-download true}}
+     :logseq.kv/graph-rtc-e2ee?              {:doc "true if it's a rtc graph with E2EE enabled"})))

deps/db/src/logseq/db/sqlite/util.cljs

@@ -137,9 +137,10 @@
           ;; Timestamp is useful as this can occur much later than :logseq.kv/graph-created-at
            (kv :logseq.kv/imported-at (common-util/time-ms))]
           (mapv
-           ;; Don't import some RTC related entities
            (fn [db-ident] [:db/retractEntity db-ident])
-           [:logseq.kv/graph-uuid
-            :logseq.kv/graph-local-tx
-            :logseq.kv/remote-schema-version
-            :logseq.kv/graph-text-embedding-model-name])))
+           [:logseq.kv/graph-uuid       ;rtc related
+            :logseq.kv/graph-local-tx   ;rtc related
+            :logseq.kv/remote-schema-version ;rtc related
+            :logseq.kv/graph-rtc-e2ee?  ;rtc related
+            :logseq.kv/graph-text-embedding-model-name ;embedding
+            ])))

src/main/frontend/worker/rtc/client.cljs

@@ -145,7 +145,6 @@
                          :repo repo
                          :graph-uuid graph-uuid
                          :remote-schema-version max-remote-schema-version}))
-          (assert @*aes-key)
           (m/? (task--apply-remote-updates-from-apply-ops
                 init-request-resp graph-uuid repo conn date-formatter @*aes-key add-log-fn))))
       ws)))
@@ -445,6 +444,7 @@
 
 (defn- task--encrypt-remote-ops
   [aes-key remote-ops]
+  (assert aes-key)
   (let [encrypt-attr-set (conj rtc-const/encrypt-attr-set :page-name)]
     (m/sp
       (loop [[remote-op & rest-remote-ops] remote-ops
@@ -464,7 +464,7 @@
                 (recur rest-remote-ops
                        (conj result [op-type (assoc op-value :av-coll av-coll*)])))
 
-;; else
+              ;; else
               (recur rest-remote-ops (conj result remote-op)))))))))
 
 (defn new-task--push-local-ops
@@ -483,11 +483,13 @@
                       other-remote-ops)]
       (when-let [ops-for-remote (rtc-schema/to-ws-ops-decoder remote-ops)]
         (let [local-tx (client-op/get-local-tx repo)
-              encrypted-remote-ops (m/? (task--encrypt-remote-ops aes-key ops-for-remote))
+              ops-for-remote* (if aes-key
+                                (m/? (task--encrypt-remote-ops aes-key ops-for-remote))
+                                ops-for-remote)
               r (try
                   (let [message (cond-> {:action "apply-ops"
                                          :graph-uuid graph-uuid :schema-version (str major-schema-version)
-                                         :ops encrypted-remote-ops :t-before local-tx}
+                                         :ops ops-for-remote* :t-before local-tx}
                                   (true? @*remote-profile?) (assoc :profile true))
                         r (m/? (ws-util/send&recv get-ws-create-task message))]
                     (r.throttle/add-rtc-api-call-record! message)

src/main/frontend/worker/rtc/core.cljs

@@ -190,14 +190,15 @@
       *online-users)))
 
 (defn- task--update-*aes-key
-  [get-ws-create-task user-uuid graph-uuid *aes-key]
+  [get-ws-create-task db user-uuid graph-uuid *aes-key]
   (m/sp
-    (let [aes-key (m/? (rtc-crypt/task--get-aes-key get-ws-create-task user-uuid graph-uuid))]
-      (when (nil? aes-key)
-        (throw (ex-info "not found aes-key" {:type :rtc.exception/not-found-graph-aes-key
-                                             :graph-uuid graph-uuid
-                                             :user-uuid user-uuid})))
-      (reset! *aes-key aes-key))))
+    (when (ldb/get-graph-rtc-e2ee? db)
+      (let [aes-key (m/? (rtc-crypt/task--get-aes-key get-ws-create-task user-uuid graph-uuid))]
+        (when (nil? aes-key)
+          (throw (ex-info "not found aes-key" {:type :rtc.exception/not-found-graph-aes-key
+                                               :graph-uuid graph-uuid
+                                               :user-uuid user-uuid})))
+        (reset! *aes-key aes-key)))))
 
 (declare new-task--inject-users-info)
 (defn- create-rtc-loop
@@ -239,7 +240,7 @@
         (try
           (log/info :rtc :loop-starting)
           ;; init run to open a ws
-          (m/? (task--update-*aes-key get-ws-create-task0 user-uuid graph-uuid *aes-key))
+          (m/? (task--update-*aes-key get-ws-create-task0 @conn user-uuid graph-uuid *aes-key))
           (m/? get-ws-create-task)
           ;; NOTE: Set dfv after ws connection is established,
           ;; ensuring the ws connection is already up when the cloud-icon turns green.
@@ -367,6 +368,8 @@
                          :fail (fn [e]
                                  (reset! *last-stop-exception e)
                                  (log/info :rtc-loop-task e)
+                                 (when-not (instance? Cancelled e)
+                                   (println (.-stack e)))
                                  (when (= :rtc.exception/ws-timeout (some-> e ex-data :type))
                                    ;; if fail reason is websocket-timeout, try to restart rtc
                                    (worker-state/<invoke-main-thread :thread-api/rtc-start-request repo))))

src/main/frontend/worker/rtc/db.cljs

@@ -9,7 +9,8 @@
   (when-let [conn (worker-state/get-datascript-conn repo)]
     (ldb/transact! conn [[:db/retractEntity :logseq.kv/graph-uuid]
                          [:db/retractEntity :logseq.kv/graph-local-tx]
-                         [:db/retractEntity :logseq.kv/remote-schema-version]])))
+                         [:db/retractEntity :logseq.kv/remote-schema-version]
+                         [:db/retractEntity :logseq.kv/graph-rtc-e2ee?]])))
 
 (defn reset-client-op-conn
   [repo]

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -158,6 +158,7 @@
                                                      :user-uuid user-uuid})))
 
       (let [encrypted-aes-key (c.m/<? (crypt/<encrypt-aes-key public-key aes-key))
+            _ (ldb/transact! conn [(ldb/kv :logseq.kv/graph-rtc-e2ee? true)])
             _ (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :fetching-presigned-put-url
                                                           :message "fetching presigned put-url"})
             [{:keys [url key]} all-blocks-str]

src/main/frontend/worker/rtc/remote_update.cljs

@@ -605,25 +605,26 @@ so need to pull earlier remote-data from websocket."})
 
 (defn task--decrypt-blocks-in-remote-update-data
   [aes-key encrypt-attr-set remote-update-data]
+  (assert aes-key)
   (m/sp
-    (let [{affected-blocks-map :affected-blocks refed-blocks :refed-blocks} remote-update-data
-          affected-blocks-map'
-          (loop [[[block-uuid affected-block] & rest-affected-blocks] affected-blocks-map
-                 affected-blocks-map-result {}]
-            (if-not block-uuid
-              affected-blocks-map-result
-              (let [affected-block' (c.m/<? (crypt/<decrypt-map aes-key encrypt-attr-set affected-block))]
-                (recur rest-affected-blocks (assoc affected-blocks-map-result block-uuid affected-block')))))
-          refed-blocks'
-          (loop [[refed-block & rest-refed-blocks] refed-blocks
-                 refed-blocks-result []]
-            (if-not refed-block
-              refed-blocks-result
-              (let [refed-block' (c.m/<? (crypt/<decrypt-map aes-key encrypt-attr-set refed-block))]
-                (recur rest-refed-blocks (conj refed-blocks-result refed-block')))))]
-      (assoc remote-update-data
-             :affected-blocks affected-blocks-map'
-             :refed-blocks refed-blocks'))))
+   (let [{affected-blocks-map :affected-blocks refed-blocks :refed-blocks} remote-update-data
+         affected-blocks-map'
+         (loop [[[block-uuid affected-block] & rest-affected-blocks] affected-blocks-map
+                affected-blocks-map-result {}]
+           (if-not block-uuid
+             affected-blocks-map-result
+             (let [affected-block' (c.m/<? (crypt/<decrypt-map aes-key encrypt-attr-set affected-block))]
+               (recur rest-affected-blocks (assoc affected-blocks-map-result block-uuid affected-block')))))
+         refed-blocks'
+         (loop [[refed-block & rest-refed-blocks] refed-blocks
+                refed-blocks-result []]
+           (if-not refed-block
+             refed-blocks-result
+             (let [refed-block' (c.m/<? (crypt/<decrypt-map aes-key encrypt-attr-set refed-block))]
+               (recur rest-refed-blocks (conj refed-blocks-result refed-block')))))]
+     (assoc remote-update-data
+            :affected-blocks affected-blocks-map'
+            :refed-blocks refed-blocks'))))
 
 (defn apply-remote-update-check
   "If the check passes, return true"
@@ -662,9 +663,11 @@ so need to pull earlier remote-data from websocket."})
   (m/sp
     (when (apply-remote-update-check repo remote-update-event add-log-fn)
       (let [remote-update-data (:value remote-update-event)
-            remote-update-data (m/? (task--decrypt-blocks-in-remote-update-data
-                                     aes-key rtc-const/encrypt-attr-set
-                                     remote-update-data))
+            remote-update-data (if aes-key
+                                 (m/? (task--decrypt-blocks-in-remote-update-data
+                                       aes-key rtc-const/encrypt-attr-set
+                                       remote-update-data))
+                                 remote-update-data)
             remote-t (:t remote-update-data)
             {affected-blocks-map :affected-blocks refed-blocks :refed-blocks} remote-update-data
             {:keys [remove-ops-map move-ops-map update-ops-map update-page-ops-map remove-page-ops-map]}