Tienson Qin 1 месяц назад
Родитель
Сommit
1f71f9e78e

+ 87 - 0
deps/db-sync/src/logseq/db_sync/index.cljs

@@ -20,6 +20,14 @@
                         "email_verified INTEGER,"
                         "username TEXT"
                         ");"))
+   (common/<d1-run db
+                   (str "create table if not exists user_rsa_keys ("
+                        "user_id TEXT primary key,"
+                        "public_key TEXT,"
+                        "encrypted_private_key TEXT,"
+                        "created_at INTEGER,"
+                        "updated_at INTEGER"
+                        ");"))
    (common/<d1-run db
                    (str "create table if not exists graph_members ("
                         "user_id TEXT,"
@@ -29,6 +37,15 @@
                         "created_at INTEGER,"
                         "primary key (user_id, graph_id),"
                         "check (role in ('manager', 'member'))"
+                        ");"))
+   (common/<d1-run db
+                   (str "create table if not exists graph_aes_keys ("
+                        "graph_id TEXT,"
+                        "user_id TEXT,"
+                        "encrypted_aes_key TEXT,"
+                        "created_at INTEGER,"
+                        "updated_at INTEGER,"
+                        "primary key (graph_id, user_id)"
                         ");"))))
 
 (defn <index-list [db user-id]
@@ -120,6 +137,76 @@
       (when row
         (aget row "id")))))
 
+(defn <user-rsa-key-pair-upsert!
+  [db user-id public-key encrypted-private-key]
+  (when (string? user-id)
+    (let [now (common/now-ms)]
+      (common/<d1-run db
+                      (str "insert into user_rsa_keys (user_id, public_key, encrypted_private_key, created_at, updated_at) "
+                           "values (?, ?, ?, ?, ?) "
+                           "on conflict(user_id) do update set "
+                           "public_key = excluded.public_key, "
+                           "encrypted_private_key = excluded.encrypted_private_key, "
+                           "updated_at = excluded.updated_at")
+                      user-id
+                      public-key
+                      encrypted-private-key
+                      now
+                      now))))
+
+(defn <user-rsa-key-pair
+  [db user-id]
+  (when (string? user-id)
+    (p/let [result (common/<d1-all db
+                                   "select public_key, encrypted_private_key from user_rsa_keys where user_id = ?"
+                                   user-id)
+            rows (common/get-sql-rows result)
+            row (first rows)]
+      (when row
+        {:public-key (aget row "public_key")
+         :encrypted-private-key (aget row "encrypted_private_key")}))))
+
+(defn <user-rsa-public-key-by-email
+  [db email]
+  (when (string? email)
+    (p/let [result (common/<d1-all db
+                                   (str "select k.public_key from user_rsa_keys k "
+                                        "left join users u on k.user_id = u.id "
+                                        "where u.email = ?")
+                                   email)
+            rows (common/get-sql-rows result)
+            row (first rows)]
+      (when row
+        (aget row "public_key")))))
+
+(defn <graph-encrypted-aes-key-upsert!
+  [db graph-id user-id encrypted-aes-key]
+  (when (and (string? graph-id) (string? user-id))
+    (let [now (common/now-ms)]
+      (common/<d1-run db
+                      (str "insert into graph_aes_keys (graph_id, user_id, encrypted_aes_key, created_at, updated_at) "
+                           "values (?, ?, ?, ?, ?) "
+                           "on conflict(graph_id, user_id) do update set "
+                           "encrypted_aes_key = excluded.encrypted_aes_key, "
+                           "updated_at = excluded.updated_at")
+                      graph-id
+                      user-id
+                      encrypted-aes-key
+                      now
+                      now))))
+
+(defn <graph-encrypted-aes-key
+  [db graph-id user-id]
+  (when (and (string? graph-id) (string? user-id))
+    (p/let [result (common/<d1-all db
+                                   "select encrypted_aes_key from graph_aes_keys where graph_id = ? and user_id = ?"
+                                   graph-id
+                                   user-id)
+            rows (common/get-sql-rows result)
+            row (first rows)]
+      (when row
+        (aget row "encrypted_aes_key")))))
+
 (defn <graph-member-upsert! [db graph-id user-id role invited-by]
   (let [now (common/now-ms)]
     (common/<d1-run db

+ 45 - 1
deps/db-sync/src/logseq/db_sync/malli_schema.cljs

@@ -159,6 +159,43 @@
    [:t-before :int]
    [:txs :string]])
 
+(def e2ee-user-key-request-schema
+  [:map
+   [:public-key :string]
+   [:encrypted-private-key :string]
+   [:reset-private-key {:optional true} :boolean]])
+
+(def e2ee-user-key-response-schema
+  [:map
+   [:public-key {:optional true} [:maybe :string]]
+   [:encrypted-private-key {:optional true} [:maybe :string]]])
+
+(def e2ee-user-public-key-response-schema
+  [:map
+   [:public-key {:optional true} [:maybe :string]]])
+
+(def e2ee-graph-aes-key-request-schema
+  [:map
+   [:encrypted-aes-key :string]])
+
+(def e2ee-graph-aes-key-response-schema
+  [:map
+   [:encrypted-aes-key {:optional true} [:maybe :string]]])
+
+(def e2ee-grant-access-entry-schema
+  [:map
+   [:user/email :string]
+   [:encrypted-aes-key :string]])
+
+(def e2ee-grant-access-request-schema
+  [:map
+   [:target-user-email+encrypted-aes-key-coll [:sequential e2ee-grant-access-entry-schema]]])
+
+(def e2ee-grant-access-response-schema
+  [:map
+   [:ok :boolean]
+   [:missing-users {:optional true} [:sequential :string]]])
+
 (def snapshot-download-response-schema
   [:map
    [:ok :boolean]
@@ -180,7 +217,10 @@
   {:graphs/create graph-create-request-schema
    :graph-members/create graph-member-create-request-schema
    :graph-members/update graph-member-update-request-schema
-   :sync/tx-batch tx-batch-request-schema})
+   :sync/tx-batch tx-batch-request-schema
+   :e2ee/user-keys e2ee-user-key-request-schema
+   :e2ee/graph-aes-key e2ee-graph-aes-key-request-schema
+   :e2ee/grant-access e2ee-grant-access-request-schema})
 
 (def http-response-schemas
   {:graphs/list graphs-list-response-schema
@@ -198,6 +238,10 @@
    :sync/snapshot-download snapshot-download-response-schema
    :sync/snapshot-upload snapshot-upload-response-schema
    :sync/admin-reset http-ok-response-schema
+   :e2ee/user-keys e2ee-user-key-response-schema
+   :e2ee/user-public-key e2ee-user-public-key-response-schema
+   :e2ee/graph-aes-key e2ee-graph-aes-key-response-schema
+   :e2ee/grant-access e2ee-grant-access-response-schema
    :assets/get asset-get-response-schema
    :assets/put http-ok-response-schema
    :assets/delete http-ok-response-schema

+ 130 - 0
deps/db-sync/src/logseq/db_sync/worker.cljs

@@ -501,6 +501,9 @@
           (string/starts-with? path "/graphs/"))
       (handle-index-fetch #js {:env env :d1 (aget env "DB")} request)
 
+      (string/starts-with? path "/e2ee")
+      (handle-index-fetch #js {:env env :d1 (aget env "DB")} request)
+
       (string/starts-with? path "/assets/")
       (if (= method "OPTIONS")
         (handle-assets request env)
@@ -1017,6 +1020,133 @@
                     (p/let [_ (index/<graph-member-delete! db graph-id member-id)]
                       (json-response :graph-members/delete {:ok true}))))))
 
+            (and (= method "GET")
+                 (= ["e2ee" "user-keys"] parts))
+            (let [user-id (aget claims "sub")]
+              (if (string? user-id)
+                (p/let [pair (index/<user-rsa-key-pair db user-id)]
+                  (json-response :e2ee/user-keys (or pair {})))
+                (unauthorized)))
+
+            (and (= method "POST")
+                 (= ["e2ee" "user-keys"] parts))
+            (.then (common/read-json request)
+                   (fn [result]
+                     (if (nil? result)
+                       (bad-request "missing body")
+                       (let [body (js->clj result :keywordize-keys true)
+                             body (coerce-http-request :e2ee/user-keys body)
+                             user-id (aget claims "sub")]
+                         (cond
+                           (not (string? user-id))
+                           (unauthorized)
+
+                           (nil? body)
+                           (bad-request "invalid body")
+
+                           :else
+                           (let [{:keys [public-key encrypted-private-key]} body]
+                             (p/let [_ (index/<user-rsa-key-pair-upsert! db user-id public-key encrypted-private-key)]
+                               (json-response :e2ee/user-keys {:public-key public-key
+                                                               :encrypted-private-key encrypted-private-key}))))))))
+
+            (and (= method "GET")
+                 (= ["e2ee" "user-public-key"] parts))
+            (let [email (.get (.-searchParams url) "email")]
+              (p/let [public-key (index/<user-rsa-public-key-by-email db email)]
+                (json-response :e2ee/user-public-key
+                               (cond-> {}
+                                 (some? public-key)
+                                 (assoc :public-key public-key)))))
+
+            (and (= method "GET")
+                 (= 4 (count parts))
+                 (= "e2ee" (first parts))
+                 (= "graphs" (nth parts 1))
+                 (= "aes-key" (nth parts 3)))
+            (let [graph-id (nth parts 2)
+                  user-id (aget claims "sub")]
+              (cond
+                (not (string? user-id))
+                (unauthorized)
+
+                :else
+                (p/let [access? (index/<user-has-access-to-graph? db graph-id user-id)]
+                  (if (not access?)
+                    (forbidden)
+                    (p/let [encrypted-aes-key (index/<graph-encrypted-aes-key db graph-id user-id)]
+                      (json-response :e2ee/graph-aes-key (cond-> {}
+                                                           (some? encrypted-aes-key)
+                                                           (assoc :encrypted-aes-key encrypted-aes-key))))))))
+
+            (and (= method "POST")
+                 (= 4 (count parts))
+                 (= "e2ee" (first parts))
+                 (= "graphs" (nth parts 1))
+                 (= "aes-key" (nth parts 3)))
+            (let [graph-id (nth parts 2)
+                  user-id (aget claims "sub")]
+              (cond
+                (not (string? user-id))
+                (unauthorized)
+
+                :else
+                (.then (common/read-json request)
+                       (fn [result]
+                         (if (nil? result)
+                           (bad-request "missing body")
+                           (let [body (js->clj result :keywordize-keys true)
+                                 body (coerce-http-request :e2ee/graph-aes-key body)]
+                             (if (nil? body)
+                               (bad-request "invalid body")
+                               (p/let [access? (index/<user-has-access-to-graph? db graph-id user-id)]
+                                 (if (not access?)
+                                   (forbidden)
+                                   (let [{:keys [encrypted-aes-key]} body]
+                                     (p/let [_ (index/<graph-encrypted-aes-key-upsert! db graph-id user-id encrypted-aes-key)]
+                                       (json-response :e2ee/graph-aes-key {:encrypted-aes-key encrypted-aes-key}))))))))))))
+
+            (and (= method "POST")
+                 (= 4 (count parts))
+                 (= "e2ee" (first parts))
+                 (= "graphs" (nth parts 1))
+                 (= "grant-access" (nth parts 3)))
+            (let [graph-id (nth parts 2)
+                  user-id (aget claims "sub")]
+              (cond
+                (not (string? user-id))
+                (unauthorized)
+
+                :else
+                (.then (common/read-json request)
+                       (fn [result]
+                         (if (nil? result)
+                           (bad-request "missing body")
+                           (let [body (js->clj result :keywordize-keys true)
+                                 body (coerce-http-request :e2ee/grant-access body)]
+                             (if (nil? body)
+                               (bad-request "invalid body")
+                               (p/let [manager? (index/<user-is-manager? db graph-id user-id)]
+                                 (if (not manager?)
+                                   (forbidden)
+                                   (let [entries (:target-user-email+encrypted-aes-key-coll body)
+                                         missing (atom [])]
+                                     (p/let [_ (p/all
+                                                (map (fn [entry]
+                                                       (let [email (:user/email entry)
+                                                             encrypted-aes-key (:encrypted-aes-key entry)]
+                                                         (p/let [target-user-id (index/<user-id-by-email db email)
+                                                                 access? (and target-user-id
+                                                                              (index/<user-has-access-to-graph? db graph-id target-user-id))]
+                                                           (if (and target-user-id access?)
+                                                             (index/<graph-encrypted-aes-key-upsert! db graph-id target-user-id encrypted-aes-key)
+                                                             (swap! missing conj email)))))
+                                                     entries))]
+                                       (json-response :e2ee/grant-access
+                                                      (cond-> {:ok true}
+                                                        (seq @missing)
+                                                        (assoc :missing-users @missing))))))))))))))
+
             (and (= method "DELETE")
                  (= 2 (count parts))
                  (= "graphs" (first parts)))

+ 118 - 1
deps/db-sync/test/logseq/db_sync/index_test.cljs

@@ -25,6 +25,22 @@
 (defn- run-sql! [state sql args]
   (record-exec! state sql)
   (cond
+    (string/includes? sql "insert into user_rsa_keys")
+    (let [[user-id public-key encrypted-private-key created-at updated-at] args]
+      (swap! state update :user-keys assoc user-id {:user-id user-id
+                                                    :public-key public-key
+                                                    :encrypted-private-key encrypted-private-key
+                                                    :created-at created-at
+                                                    :updated-at updated-at}))
+
+    (string/includes? sql "insert into graph_aes_keys")
+    (let [[graph-id user-id encrypted-aes-key created-at updated-at] args]
+      (swap! state update :graph-keys assoc [graph-id user-id] {:graph-id graph-id
+                                                                :user-id user-id
+                                                                :encrypted-aes-key encrypted-aes-key
+                                                                :created-at created-at
+                                                                :updated-at updated-at}))
+
     (string/includes? sql "insert into users")
     (let [[user-id email email-verified username] args]
       (swap! state update :users assoc user-id {:id user-id
@@ -85,6 +101,30 @@
 (defn- all-sql [state sql args]
   (record-exec! state sql)
   (cond
+    (string/includes? sql "from user_rsa_keys")
+    (if (string/includes? sql "left join users")
+      (let [[email] args
+            user-id (some (fn [[_ row]]
+                            (when (= email (:email row))
+                              (:id row)))
+                          (:users @state))
+            row (when user-id (get-in @state [:user-keys user-id]))]
+        (if row
+          (js-rows [row])
+          (js-rows [])))
+      (let [[user-id] args
+            row (get-in @state [:user-keys user-id])]
+        (if row
+          (js-rows [row])
+          (js-rows []))))
+
+    (string/includes? sql "from graph_aes_keys")
+    (let [[graph-id user-id] args
+          row (get-in @state [:graph-keys [graph-id user-id]])]
+      (if row
+        (js-rows [row])
+        (js-rows [])))
+
     (string/includes? sql "from graph_members where graph_id")
     (let [graph-id (first args)
           members (->> (:graph-members @state)
@@ -154,6 +194,8 @@
   (async done
          (let [state (atom {:executed []
                             :users {}
+                            :user-keys {}
+                            :graph-keys {}
                             :graph-members {}
                             :graphs {}})
                db (make-d1 state)]
@@ -161,7 +203,9 @@
                (p/then (fn [_]
                          (let [sqls (:executed @state)]
                            (is (some #(string/includes? % "create table if not exists users") sqls))
-                           (is (some #(string/includes? % "create table if not exists graph_members") sqls)))
+                           (is (some #(string/includes? % "create table if not exists graph_members") sqls))
+                           (is (some #(string/includes? % "create table if not exists user_rsa_keys") sqls))
+                           (is (some #(string/includes? % "create table if not exists graph_aes_keys") sqls)))
                          (done)))
                (p/catch (fn [e]
                           (is false (str e))
@@ -171,6 +215,8 @@
   (async done
          (let [state (atom {:executed []
                             :users {}
+                            :user-keys {}
+                            :graph-keys {}
                             :graph-members {}
                             :graphs {}})
                db (make-d1 state)
@@ -193,6 +239,8 @@
   (async done
          (let [state (atom {:executed []
                             :users {}
+                            :user-keys {}
+                            :graph-keys {}
                             :graph-members {}
                             :graphs {}})
                db (make-d1 state)]
@@ -215,6 +263,8 @@
   (async done
          (let [state (atom {:executed []
                             :users {}
+                            :user-keys {}
+                            :graph-keys {}
                             :graph-members {}
                             :graphs {}})
                db (make-d1 state)]
@@ -240,6 +290,8 @@
   (async done
          (let [state (atom {:executed []
                             :users {}
+                            :user-keys {}
+                            :graph-keys {}
                             :graph-members {}
                             :graphs {}})
                db (make-d1 state)]
@@ -252,3 +304,68 @@
                (p/catch (fn [e]
                           (is false (str e))
                           (done)))))))
+
+(deftest e2ee-user-rsa-key-pair-upsert-test
+  (async done
+         (let [state (atom {:executed []
+                            :users {}
+                            :user-keys {}
+                            :graph-keys {}
+                            :graph-members {}
+                            :graphs {}})
+               db (make-d1 state)]
+           (-> (p/do!
+                (index/<user-rsa-key-pair-upsert! db "user-1" "public-1" "private-1")
+                (index/<user-rsa-key-pair-upsert! db "user-1" "public-2" "private-2"))
+               (p/then (fn [_]
+                         (p/let [pair (index/<user-rsa-key-pair db "user-1")]
+                           (is (= "public-2" (:public-key pair)))
+                           (is (= "private-2" (:encrypted-private-key pair)))
+                           (done))))
+               (p/catch (fn [e]
+                          (is false (str e))
+                          (done)))))))
+
+(deftest e2ee-user-public-key-by-email-test
+  (async done
+         (let [state (atom {:executed []
+                            :users {}
+                            :user-keys {}
+                            :graph-keys {}
+                            :graph-members {}
+                            :graphs {}})
+               db (make-d1 state)
+               claims #js {"sub" "user-1"
+                           "email" "[email protected]"
+                           "email_verified" true
+                           "cognito:username" "foo"}]
+           (-> (p/do!
+                (index/<user-upsert! db claims)
+                (index/<user-rsa-key-pair-upsert! db "user-1" "public-1" "private-1"))
+               (p/then (fn [_]
+                         (p/let [public-key (index/<user-rsa-public-key-by-email db "[email protected]")]
+                           (is (= "public-1" public-key))
+                           (done))))
+               (p/catch (fn [e]
+                          (is false (str e))
+                          (done)))))))
+
+(deftest e2ee-graph-aes-key-upsert-test
+  (async done
+         (let [state (atom {:executed []
+                            :users {}
+                            :user-keys {}
+                            :graph-keys {}
+                            :graph-members {}
+                            :graphs {}})
+               db (make-d1 state)]
+           (-> (p/do!
+                (index/<graph-encrypted-aes-key-upsert! db "graph-1" "user-1" "aes-1")
+                (index/<graph-encrypted-aes-key-upsert! db "graph-1" "user-1" "aes-2"))
+               (p/then (fn [_]
+                         (p/let [aes-key (index/<graph-encrypted-aes-key db "graph-1" "user-1")]
+                           (is (= "aes-2" aes-key))
+                           (done))))
+               (p/catch (fn [e]
+                          (is false (str e))
+                          (done)))))))

+ 21 - 0
deps/db-sync/test/logseq/db_sync/worker_routing_test.cljs

@@ -0,0 +1,21 @@
+(ns logseq.db-sync.worker-routing-test
+  (:require [cljs.test :refer [deftest is async]]
+            [logseq.db-sync.worker :as worker]))
+
+(deftest e2ee-route-uses-index-handler-test
+  (async done
+         (let [called (atom nil)
+               req (js/Request. "http://localhost/e2ee/user-keys" #js {:method "GET"})
+               env #js {:DB :db}]
+           (with-redefs [worker/handle-index-fetch (fn [_ request]
+                                                     (reset! called request)
+                                                     (js/Response. "ok"))]
+             (let [resp (#'worker/handle-worker-fetch req env)]
+               (-> (.text resp)
+                   (.then (fn [text]
+                            (is (= "ok" text))
+                            (is (some? @called))
+                            (done)))
+                   (.catch (fn [e]
+                             (is false (str e))
+                             (done)))))))))

+ 18 - 0
docs/agent-guide/db-sync/protocol.md

@@ -62,6 +62,24 @@
 - `DELETE /graphs/:graph-id`
   - Delete graph and reset data. Response: `{"graph-id":"...","deleted":true}` or `400` (missing graph id).
 
+### E2EE (index DO)
+- `GET /e2ee/user-keys`
+  - Fetch current user's RSA key pair. Response: `{"public-key":"<transit>","encrypted-private-key":"<transit>"}` or `{}` when missing.
+- `POST /e2ee/user-keys`
+  - Upsert current user's RSA key pair. Body: `{"public-key":"<transit>","encrypted-private-key":"<transit>","reset-private-key":false?}`.
+  - Response mirrors the stored keys: `{"public-key":"<transit>","encrypted-private-key":"<transit>"}`.
+- `GET /e2ee/user-public-key?email=<email>`
+  - Fetch a user's RSA public key by email. Response: `{"public-key":"<transit>"}` or `{}` when missing.
+- `GET /e2ee/graphs/:graph-id/aes-key`
+  - Fetch current user's encrypted graph AES key. Response: `{"encrypted-aes-key":"<transit>"}` or `{}` when missing.
+- `POST /e2ee/graphs/:graph-id/aes-key`
+  - Upsert current user's encrypted graph AES key. Body: `{"encrypted-aes-key":"<transit>"}`.
+  - Response: `{"encrypted-aes-key":"<transit>"}`.
+- `POST /e2ee/graphs/:graph-id/grant-access`
+  - Manager-only. Upsert encrypted graph AES keys for members.
+  - Body: `{"target-user-email+encrypted-aes-key-coll":[{"user/email":"<email>","encrypted-aes-key":"<transit>"}...]}`.
+  - Response: `{"ok":true,"missing-users":["<email>", ...]?}`.
+
 ### Sync (per-graph DO, via `/sync/:graph-id/...`)
 - `GET /sync/:graph-id/health`
   - Health check. Response: `{"ok":true}`.

+ 2 - 7
src/main/frontend/components/repo.cljs

@@ -469,17 +469,12 @@
              db-name (util/trim-safe (.-value (rum/deref input-ref)))]
          (when (and cloud? refresh-token token user-uuid
                     (not e2ee-rsa-key-ensured?))
-           (p/do!
-            (state/<invoke-db-worker :thread-api/init-user-rsa-key-pair
-                                     token
-                                     refresh-token
-                                     user-uuid))
-           (-> (p/let [rsa-key-pair (state/<invoke-db-worker :thread-api/get-user-rsa-key-pair token user-uuid)]
+           (-> (p/let [rsa-key-pair (state/<invoke-db-worker :thread-api/db-sync-ensure-user-rsa-keys)]
                  (set-e2ee-rsa-key-ensured? (some? rsa-key-pair))
                  (when rsa-key-pair
                    (when db-name (new-db-f db-name))))
                (p/catch (fn [e]
-                          (log/error :get-user-rsa-key-pair e)
+                          (log/error :db-sync/ensure-user-rsa-keys-failed e)
                           e))))))
      [cloud?])
 

+ 23 - 5
src/main/frontend/handler/db_based/db_sync.cljs

@@ -42,6 +42,18 @@
 (defn- decode-snapshot-rows [bytes]
   (sqlite-util/read-transit-str (.decode snapshot-text-decoder (->uint8 bytes))))
 
+(defn- snapshot-rows-e2ee?
+  [rows]
+  (boolean
+   (some (fn [[_ content _]]
+           (try
+             (let [data (sqlite-util/read-transit-str content)]
+               (and (map? data)
+                    (= :logseq.kv/graph-rtc-e2ee? (:db/ident data))))
+             (catch :default _
+               false)))
+         rows)))
+
 (defn- frame-len [^js data offset]
   (let [view (js/DataView. (.-buffer data) offset 4)]
     (.getUint32 view 0 false)))
@@ -267,10 +279,11 @@
                                 total' (+ total (count rows))
                                 total-rows' (into total-rows rows)]
                             (when (seq total-rows')
-                              (p/do!
-                               (state/<invoke-db-worker :thread-api/db-sync-import-kvs-rows
-                                                        graph total-rows' true)
-                               (state/<invoke-db-worker :thread-api/db-sync-finalize-kvs-import graph remote-tx)))
+                              (let [e2ee? (snapshot-rows-e2ee? total-rows')]
+                                (p/do!
+                                 (state/<invoke-db-worker :thread-api/db-sync-import-kvs-rows
+                                                          graph total-rows' true graph-uuid e2ee?)
+                                 (state/<invoke-db-worker :thread-api/db-sync-finalize-kvs-import graph remote-tx))))
                             total')
                           (let [value (.-value chunk)
                                 {:keys [rows buffer]} (parse-framed-chunk buffer value)
@@ -337,7 +350,12 @@
                              {:method "POST"
                               :headers {"content-type" "application/json"}
                               :body (js/JSON.stringify (clj->js body))}
-                             {:response-schema :graph-members/create})]
+                             {:response-schema :graph-members/create})
+               repo (state/get-current-repo)
+               e2ee? (ldb/get-graph-rtc-e2ee? (db/get-db))
+               _ (when (and repo e2ee?)
+                   (state/<invoke-db-worker :thread-api/db-sync-grant-graph-access
+                                            repo graph-uuid email))]
          (notification/show! "Invitation sent!" :success))
        (p/catch (fn [e]
                   (notification/show! "Something wrong, please try again." :error)

+ 384 - 56
src/main/frontend/worker/db_sync.cljs

@@ -1,10 +1,14 @@
 (ns frontend.worker.db-sync
   "Simple db-sync client based on promesa + WebSocket."
-  (:require [clojure.data :as data]
+  (:require ["/frontend/idbkv" :as idb-keyval]
+            [clojure.data :as data]
             [clojure.string :as string]
             [datascript.core :as d]
+            [frontend.common.crypt :as crypt]
+            [frontend.worker-common.util :as worker-util]
             [frontend.worker.handler.page :as worker-page]
             [frontend.worker.rtc.client-op :as client-op]
+            [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.shared-service :as shared-service]
             [frontend.worker.state :as worker-state]
             [lambdaisland.glogi :as log]
@@ -21,6 +25,8 @@
             [promesa.core :as p]))
 
 (defonce *repo->latest-remote-tx (atom {}))
+(defonce ^:private *repo->aes-key (atom {}))
+(defonce ^:private e2ee-store (delay (idb-keyval/newStore "localforage" "keyvaluepairs" 2)))
 
 (defn- current-client
   [repo]
@@ -33,20 +39,21 @@
 
 (defn- sync-counts
   [repo]
-  (let [pending-local (when-let [conn (client-ops-conn repo)]
-                        (count (d/datoms @conn :avet :db-sync/created-at)))
-        pending-asset (client-op/get-unpushed-asset-ops-count repo)
-        local-tx (client-op/get-local-tx repo)
-        remote-tx (get @*repo->latest-remote-tx repo)
-        pending-server (when (and (number? local-tx) (number? remote-tx))
-                         (max 0 (- remote-tx local-tx)))
-        graph-uuid (client-op/get-graph-uuid repo)]
-    {:pending-local pending-local
-     :pending-asset pending-asset
-     :pending-server pending-server
-     :local-tx local-tx
-     :remote-tx remote-tx
-     :graph-uuid graph-uuid}))
+  (when (worker-state/get-datascript-conn repo)
+    (let [pending-local (when-let [conn (client-ops-conn repo)]
+                          (count (d/datoms @conn :avet :db-sync/created-at)))
+          pending-asset (client-op/get-unpushed-asset-ops-count repo)
+          local-tx (client-op/get-local-tx repo)
+          remote-tx (get @*repo->latest-remote-tx repo)
+          pending-server (when (and (number? local-tx) (number? remote-tx))
+                           (max 0 (- remote-tx local-tx)))
+          graph-uuid (client-op/get-graph-uuid repo)]
+      {:pending-local pending-local
+       :pending-asset pending-asset
+       :pending-server pending-server
+       :local-tx local-tx
+       :remote-tx remote-tx
+       :graph-uuid graph-uuid})))
 
 (defn- normalize-online-users
   [users]
@@ -309,6 +316,333 @@
                          :url url
                          :body body}))))))
 
+(def ^:private invalid-transit ::invalid-transit)
+
+(declare encrypt-snapshot-rows decrypt-snapshot-rows)
+
+(defn- try-read-transit [value]
+  (try
+    (ldb/read-transit-str value)
+    (catch :default _
+      invalid-transit)))
+
+(defn- graph-e2ee?
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (true? (ldb/get-graph-rtc-e2ee? @conn))))
+
+(defn- user-uuid []
+  (some-> (worker-state/get-id-token) worker-util/parse-jwt :sub))
+
+(defn- graph-encrypted-aes-key-idb-key
+  [graph-id]
+  (str "rtc-encrypted-aes-key###" graph-id))
+
+(defn- <get-item
+  [k]
+  (assert (and k @e2ee-store))
+  (p/let [r (idb-keyval/get k @e2ee-store)]
+    (js->clj r :keywordize-keys true)))
+
+(defn- <set-item!
+  [k value]
+  (assert (and k @e2ee-store))
+  (idb-keyval/set k value @e2ee-store))
+
+(defn- e2ee-base
+  []
+  (http-base-url))
+
+(defn- <fetch-user-rsa-key-pair-raw
+  [base]
+  (fetch-json (str base "/e2ee/user-keys")
+              {:method "GET"}
+              {:response-schema :e2ee/user-keys}))
+
+(defn- <upload-user-rsa-key-pair!
+  [base public-key encrypted-private-key]
+  (let [body (coerce-http-request :e2ee/user-keys
+                                  {:public-key public-key
+                                   :encrypted-private-key encrypted-private-key})]
+    (when (nil? body)
+      (fail-fast :db-sync/invalid-field {:type :e2ee/user-keys :body body}))
+    (fetch-json (str base "/e2ee/user-keys")
+                {:method "POST"
+                 :headers {"content-type" "application/json"}
+                 :body (js/JSON.stringify (clj->js body))}
+                {:response-schema :e2ee/user-keys})))
+
+(defn- <ensure-user-rsa-key-pair-raw
+  [base]
+  (p/let [existing (-> (<fetch-user-rsa-key-pair-raw base)
+                       (p/catch (fn [error]
+                                  (throw error))))]
+    (if (and (string? (:public-key existing))
+             (string? (:encrypted-private-key existing)))
+      existing
+      (p/let [{:keys [publicKey privateKey]} (crypt/<generate-rsa-key-pair)
+              {:keys [password]} (worker-state/<invoke-main-thread :thread-api/request-e2ee-password)
+              encrypted-private-key (crypt/<encrypt-private-key password privateKey)
+              exported-public-key (crypt/<export-public-key publicKey)
+              public-key-str (ldb/write-transit-str exported-public-key)
+              encrypted-private-key-str (ldb/write-transit-str encrypted-private-key)]
+        (p/let [_ (<upload-user-rsa-key-pair! base public-key-str encrypted-private-key-str)]
+          {:public-key public-key-str
+           :encrypted-private-key encrypted-private-key-str})))))
+
+(defn ensure-user-rsa-keys!
+  []
+  (let [base (e2ee-base)]
+    (when-not (string? base)
+      (fail-fast :db-sync/missing-field {:base base}))
+    (<ensure-user-rsa-key-pair-raw base)))
+
+(defn- <decrypt-private-key
+  [encrypted-private-key-str]
+  (p/let [encrypted-private-key (ldb/read-transit-str encrypted-private-key-str)
+          exported-private-key (worker-state/<invoke-main-thread
+                                :thread-api/decrypt-user-e2ee-private-key
+                                encrypted-private-key)]
+    (crypt/<import-private-key exported-private-key)))
+
+(defn- <import-public-key
+  [public-key-str]
+  (p/let [exported (ldb/read-transit-str public-key-str)]
+    (crypt/<import-public-key exported)))
+
+(defn- <fetch-user-public-key-by-email
+  [base email]
+  (fetch-json (str base "/e2ee/user-public-key?email=" (js/encodeURIComponent email))
+              {:method "GET"}
+              {:response-schema :e2ee/user-public-key}))
+
+(defn- <fetch-graph-encrypted-aes-key-raw
+  [base graph-id]
+  (fetch-json (str base "/e2ee/graphs/" graph-id "/aes-key")
+              {:method "GET"}
+              {:response-schema :e2ee/graph-aes-key}))
+
+(defn- <upsert-graph-encrypted-aes-key!
+  [base graph-id encrypted-aes-key]
+  (let [body (coerce-http-request :e2ee/graph-aes-key
+                                  {:encrypted-aes-key encrypted-aes-key})]
+    (when (nil? body)
+      (fail-fast :db-sync/invalid-field {:type :e2ee/graph-aes-key :body body}))
+    (fetch-json (str base "/e2ee/graphs/" graph-id "/aes-key")
+                {:method "POST"
+                 :headers {"content-type" "application/json"}
+                 :body (js/JSON.stringify (clj->js body))}
+                {:response-schema :e2ee/graph-aes-key})))
+
+(defn- <ensure-graph-aes-key
+  [repo graph-id]
+  (if-not (graph-e2ee? repo)
+    (p/resolved nil)
+    (if-let [cached (get @*repo->aes-key repo)]
+      (p/resolved cached)
+      (let [base (e2ee-base)
+            user-id (user-uuid)]
+        (when-not (and (string? base) (string? user-id))
+          (fail-fast :db-sync/missing-field {:base base :user-id user-id :graph-id graph-id}))
+        (p/let [{:keys [public-key encrypted-private-key]} (<ensure-user-rsa-key-pair-raw base)
+                public-key' (when (string? public-key) (<import-public-key public-key))
+                private-key' (when (string? encrypted-private-key) (<decrypt-private-key encrypted-private-key))
+                local-encrypted (when graph-id
+                                  (<get-item (graph-encrypted-aes-key-idb-key graph-id)))
+                remote-encrypted (when (and (nil? local-encrypted) graph-id)
+                                   (p/let [resp (<fetch-graph-encrypted-aes-key-raw base graph-id)]
+                                     (when-let [encrypted-aes-key (:encrypted-aes-key resp)]
+                                       (ldb/read-transit-str encrypted-aes-key))))
+                encrypted-aes-key (or local-encrypted remote-encrypted)
+                aes-key (if encrypted-aes-key
+                          (crypt/<decrypt-aes-key private-key' encrypted-aes-key)
+                          (p/let [aes-key (crypt/<generate-aes-key)
+                                  encrypted (crypt/<encrypt-aes-key public-key' aes-key)
+                                  encrypted-str (ldb/write-transit-str encrypted)
+                                  _ (<upsert-graph-encrypted-aes-key! base graph-id encrypted-str)
+                                  _ (<set-item! (graph-encrypted-aes-key-idb-key graph-id) encrypted)]
+                            aes-key))
+                _ (when (and graph-id encrypted-aes-key (nil? local-encrypted))
+                    (<set-item! (graph-encrypted-aes-key-idb-key graph-id) encrypted-aes-key))]
+          (swap! *repo->aes-key assoc repo aes-key)
+          aes-key)))))
+
+(defn- <fetch-graph-aes-key-for-download
+  [repo graph-id]
+  (let [base (e2ee-base)]
+    (when-not (and (string? base) (string? graph-id))
+      (fail-fast :db-sync/missing-field {:base base :graph-id graph-id}))
+    (p/let [{:keys [public-key encrypted-private-key]} (<fetch-user-rsa-key-pair-raw base)]
+      (when-not (and (string? public-key) (string? encrypted-private-key))
+        (fail-fast :db-sync/missing-field {:graph-id graph-id :field :user-rsa-key-pair}))
+      (p/let [private-key (<decrypt-private-key encrypted-private-key)
+              local-encrypted (<get-item (graph-encrypted-aes-key-idb-key graph-id))
+              remote-encrypted (when (nil? local-encrypted)
+                                 (p/let [resp (<fetch-graph-encrypted-aes-key-raw base graph-id)]
+                                   (when-let [encrypted-aes-key (:encrypted-aes-key resp)]
+                                     (ldb/read-transit-str encrypted-aes-key))))
+              encrypted-aes-key (or local-encrypted remote-encrypted)]
+        (when-not encrypted-aes-key
+          (fail-fast :db-sync/missing-field {:graph-id graph-id :field :encrypted-aes-key}))
+        (when (and encrypted-aes-key (nil? local-encrypted))
+          (<set-item! (graph-encrypted-aes-key-idb-key graph-id) encrypted-aes-key))
+        (p/let [aes-key (crypt/<decrypt-aes-key private-key encrypted-aes-key)]
+          (swap! *repo->aes-key assoc repo aes-key)
+          aes-key)))))
+
+(defn <decrypt-kvs-rows
+  [repo graph-id rows e2ee?]
+  (if-not (true? e2ee?)
+    (p/resolved rows)
+    (p/let [aes-key (<fetch-graph-aes-key-for-download repo graph-id)
+            _ (when (nil? aes-key)
+                (fail-fast :db-sync/missing-field {:repo repo :field :aes-key}))
+            rows* (decrypt-snapshot-rows aes-key rows)]
+      rows*)))
+
+(defn- <grant-graph-access!
+  [repo graph-id target-email]
+  (if-not (graph-e2ee? repo)
+    (p/resolved nil)
+    (let [base (e2ee-base)]
+      (when-not (string? base)
+        (fail-fast :db-sync/missing-field {:base base :graph-id graph-id}))
+      (p/let [aes-key (<ensure-graph-aes-key repo graph-id)
+              _ (when (nil? aes-key)
+                  (fail-fast :db-sync/missing-field {:repo repo :field :aes-key}))
+              resp (<fetch-user-public-key-by-email base target-email)
+              public-key-str (:public-key resp)]
+        (if-not (string? public-key-str)
+          (fail-fast :db-sync/missing-field {:repo repo :field :public-key :email target-email})
+          (p/let [public-key (<import-public-key public-key-str)
+                  encrypted (crypt/<encrypt-aes-key public-key aes-key)
+                  encrypted-str (ldb/write-transit-str encrypted)
+                  body (coerce-http-request :e2ee/grant-access
+                                            {:target-user-email+encrypted-aes-key-coll
+                                             [{:user/email target-email
+                                               :encrypted-aes-key encrypted-str}]})
+                  _ (when (nil? body)
+                      (fail-fast :db-sync/invalid-field {:type :e2ee/grant-access :body body}))
+                  _ (fetch-json (str base "/e2ee/graphs/" graph-id "/grant-access")
+                                {:method "POST"
+                                 :headers {"content-type" "application/json"}
+                                 :body (js/JSON.stringify (clj->js body))}
+                                {:response-schema :e2ee/grant-access})]
+            nil))))))
+
+(defn grant-graph-access!
+  [repo graph-id target-email]
+  (<grant-graph-access! repo graph-id target-email))
+
+(defn- <encrypt-text-value
+  [aes-key value]
+  (p/let [text (ldb/write-transit-str value)
+          encrypted (crypt/<encrypt-text aes-key text)]
+    (ldb/write-transit-str encrypted)))
+
+(defn- <decrypt-text-value
+  [aes-key value]
+  (if-not (string? value)
+    (p/resolved value)
+    (let [decoded (try-read-transit value)]
+      (if (= decoded invalid-transit)
+        (p/resolved value)
+        (p/let [decrypted (crypt/<decrypt-text-if-encrypted aes-key decoded)]
+          (if decrypted
+            (ldb/read-transit-str decrypted)
+            value))))))
+
+(defn- encrypt-tx-item
+  [aes-key item]
+  (cond
+    (and (vector? item) (<= 4 (count item)))
+    (let [attr (nth item 2)
+          v (nth item 3)]
+      (if (and (contains? rtc-const/encrypt-attr-set attr)
+               (string? v))
+        (p/let [v' (<encrypt-text-value aes-key v)]
+          (assoc item 3 v'))
+        (p/resolved item)))
+
+    (map? item)
+    (let [attr (:a item)
+          v (:v item)]
+      (if (and (contains? rtc-const/encrypt-attr-set attr)
+               (string? v))
+        (p/let [v' (<encrypt-text-value aes-key v)]
+          (assoc item :v v'))
+        (p/resolved item)))
+
+    :else
+    (p/resolved item)))
+
+(defn- decrypt-tx-item
+  [aes-key item]
+  (cond
+    (and (vector? item) (<= 4 (count item)))
+    (let [attr (nth item 2)
+          v (nth item 3)]
+      (if (and (contains? rtc-const/encrypt-attr-set attr)
+               (string? v))
+        (p/let [v' (<decrypt-text-value aes-key v)]
+          (assoc item 3 v'))
+        (p/resolved item)))
+
+    (map? item)
+    (let [attr (:a item)
+          v (:v item)]
+      (if (and (contains? rtc-const/encrypt-attr-set attr)
+               (string? v))
+        (p/let [v' (<decrypt-text-value aes-key v)]
+          (assoc item :v v'))
+        (p/resolved item)))
+
+    :else
+    (p/resolved item)))
+
+(defn- encrypt-tx-data
+  [aes-key tx-data]
+  (if-not (seq tx-data)
+    (p/resolved [])
+    (p/let [items (p/all (mapv (fn [item] (encrypt-tx-item aes-key item)) tx-data))]
+      (vec items))))
+
+(defn- decrypt-tx-data
+  [aes-key tx-data]
+  (if-not (seq tx-data)
+    (p/resolved [])
+    (p/let [items (p/all (mapv (fn [item] (decrypt-tx-item aes-key item)) tx-data))]
+      (vec items))))
+
+(defn- encrypt-snapshot-rows
+  [aes-key rows]
+  (if-not (seq rows)
+    (p/resolved [])
+    (p/let [items (p/all
+                   (mapv (fn [[addr content addresses]]
+                           (let [data (try-read-transit content)]
+                             (if (and (not= data invalid-transit) (map? data))
+                               (p/let [data' (crypt/<encrypt-map aes-key rtc-const/encrypt-attr-set data)
+                                       content' (ldb/write-transit-str data')]
+                                 [addr content' addresses])
+                               (p/resolved [addr content addresses]))))
+                         rows))]
+      (vec items))))
+
+(defn- decrypt-snapshot-rows
+  [aes-key rows]
+  (if-not (seq rows)
+    (p/resolved [])
+    (p/let [items (p/all
+                   (mapv (fn [[addr content addresses]]
+                           (let [data (try-read-transit content)]
+                             (if (and (not= data invalid-transit) (map? data))
+                               (p/let [data' (crypt/<decrypt-map aes-key rtc-const/encrypt-attr-set data)
+                                       content' (ldb/write-transit-str data')]
+                                 [addr content' addresses])
+                               (p/resolved [addr content addresses]))))
+                         rows))]
+      (vec items))))
 (defn- require-asset-field
   [repo field value context]
   (when (or (nil? value) (and (string? value) (string/blank? value)))
@@ -414,10 +748,16 @@
                   ;; (prn :debug :before-keep-last-update txs)
                   ;; (prn :debug :upload :tx-data tx-data)
                   (when (seq txs)
-                    (reset! (:inflight client) tx-ids)
-                    (send! ws {:type "tx/batch"
-                               :t-before local-tx
-                               :txs (sqlite-util/write-transit-str tx-data)})))))))))))
+                    (p/let [aes-key (<ensure-graph-aes-key repo (:graph-id client))
+                            _ (when (and (graph-e2ee? repo) (nil? aes-key))
+                                (fail-fast :db-sync/missing-field {:repo repo :field :aes-key}))
+                            tx-data* (if aes-key
+                                       (encrypt-tx-data aes-key tx-data)
+                                       (p/resolved tx-data))]
+                      (reset! (:inflight client) tx-ids)
+                      (send! ws {:type "tx/batch"
+                                 :t-before local-tx
+                                 :txs (sqlite-util/write-transit-str tx-data*)}))))))))))))
 
 (defn- ensure-client-state! [repo]
   (let [client {:repo repo
@@ -829,12 +1169,18 @@
                                          txs)
                           tx (distinct (mapcat identity txs-data))]
                       (when (seq tx)
-                        (apply-remote-tx! repo client tx
-                                          :local-tx local-tx
-                                          :remote-tx remote-tx)
-                        (client-op/update-local-tx repo remote-tx)
-                        (broadcast-rtc-state! client)
-                        (flush-pending! repo client))))
+                        (p/let [aes-key (<ensure-graph-aes-key repo (:graph-id client))
+                                _ (when (and (graph-e2ee? repo) (nil? aes-key))
+                                    (fail-fast :db-sync/missing-field {:repo repo :field :aes-key}))
+                                tx* (if aes-key
+                                      (decrypt-tx-data aes-key tx)
+                                      (p/resolved tx))]
+                          (apply-remote-tx! repo client tx*
+                                            :local-tx local-tx
+                                            :remote-tx remote-tx)
+                          (client-op/update-local-tx repo remote-tx)
+                          (broadcast-rtc-state! client)
+                          (flush-pending! repo client)))))
         "changed" (do
                     (require-non-negative remote-tx {:repo repo :type "changed"})
                     (broadcast-rtc-state! client)
@@ -1010,40 +1356,11 @@
     (.set out bytes 4)
     out))
 
-(defn- snapshot-upload-stream [db]
-  (let [state (volatile! {:after -1 :done? false})]
-    (js/ReadableStream.
-     #js {:pull (fn [controller]
-                  (p/let [{:keys [after done?]} @state]
-                    (if done?
-                      (.close controller)
-                      (let [rows (fetch-kvs-rows db after upload-kvs-batch-size)]
-                        (if (empty? rows)
-                          (.close controller)
-                          (let [rows (normalize-snapshot-rows rows)
-                                last-addr (apply max (map first rows))
-                                done? (< (count rows) upload-kvs-batch-size)
-                                payload (encode-snapshot-rows rows)
-                                framed (frame-bytes payload)]
-                            (.enqueue controller framed)
-                            (vswap! state assoc :after last-addr :done? done?)))))))})))
-
 (defn- maybe-compress-stream [stream]
   (if (exists? js/CompressionStream)
     (.pipeThrough stream (js/CompressionStream. "gzip"))
     stream))
 
-(defn- should-buffer-snapshot-upload?
-  [base]
-  (when (string? base)
-    (try
-      (let [url (js/URL. base)
-            host (.-hostname url)]
-        (and (= "http:" (.-protocol url))
-             (contains? #{"localhost" "127.0.0.1"} host)))
-      (catch :default _
-        false))))
-
 (defn- <buffer-stream
   [stream]
   (p/let [resp (js/Response. stream)
@@ -1064,13 +1381,21 @@
         {:body buf :encoding snapshot-content-encoding})
       (p/resolved {:body frame :encoding nil}))))
 
+(defn- set-graph-e2ee-enabled!
+  [repo]
+  (when-let [conn (worker-state/get-datascript-conn repo)]
+    (ldb/transact! conn [(ldb/kv :logseq.kv/graph-rtc-e2ee? true)])))
+
 (defn upload-graph!
   [repo]
   (let [base (http-base-url)
         graph-id (get-graph-id repo)]
     (if (and (seq base) (seq graph-id))
       (if-let [db (worker-state/get-sqlite-conn repo :db)]
-        (do
+        (p/let [aes-key (<ensure-graph-aes-key repo graph-id)
+                _ (when (and (graph-e2ee? repo) (nil? aes-key))
+                    (fail-fast :db-sync/missing-field {:repo repo :field :aes-key}))]
+          (set-graph-e2ee-enabled! repo)
           (ensure-client-graph-uuid! repo graph-id)
           (p/loop [last-addr -1
                    first-batch? true]
@@ -1084,7 +1409,10 @@
                 (let [max-addr (apply max (map first rows))
                       rows (normalize-snapshot-rows rows)
                       upload-url (str base "/sync/" graph-id "/snapshot/upload?reset=" (if first-batch? "true" "false"))]
-                  (p/let [{:keys [body encoding]} (<snapshot-upload-body rows)
+                  (p/let [rows* (if aes-key
+                                  (encrypt-snapshot-rows aes-key rows)
+                                  (p/resolved rows))
+                          {:keys [body encoding]} (<snapshot-upload-body rows*)
                           headers (cond-> {"content-type" snapshot-content-type}
                                     (string? encoding) (assoc "content-encoding" encoding))
                           _ (fetch-json upload-url

+ 14 - 3
src/main/frontend/worker/db_worker.cljs

@@ -429,6 +429,14 @@
   [editing-block-uuid]
   (db-sync/update-presence! editing-block-uuid))
 
+(def-thread-api :thread-api/db-sync-grant-graph-access
+  [repo graph-id target-email]
+  (db-sync/grant-graph-access! repo graph-id target-email))
+
+(def-thread-api :thread-api/db-sync-ensure-user-rsa-keys
+  []
+  (db-sync/ensure-user-rsa-keys!))
+
 (def-thread-api :thread-api/db-sync-upload-graph
   [repo]
   (db-sync/upload-graph! repo))
@@ -607,12 +615,15 @@
   nil)
 
 (def-thread-api :thread-api/db-sync-import-kvs-rows
-  [repo rows reset?]
+  [repo rows reset? graph-id e2ee?]
   (p/let [_ (when reset?
               (close-db! repo))
+          rows* (if (true? e2ee?)
+                  (db-sync/<decrypt-kvs-rows repo graph-id rows e2ee?)
+                  (p/resolved rows))
           db (ensure-db-sync-import-db! repo reset?)]
-    (when (seq rows)
-      (upsert-addr-content! db (rows->sqlite-binds rows)))
+    (when (seq rows*)
+      (upsert-addr-content! db (rows->sqlite-binds rows*)))
     nil))
 
 (def-thread-api :thread-api/db-sync-finalize-kvs-import

+ 63 - 2
src/test/frontend/worker/db_sync_test.cljs

@@ -1,11 +1,14 @@
 (ns frontend.worker.db-sync-test
-  (:require [cljs.test :refer [deftest is testing run-test]]
+  (:require [cljs.test :refer [deftest is testing run-test async]]
             [datascript.core :as d]
+            [frontend.common.crypt :as crypt]
             [frontend.worker.db-sync :as db-sync]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.state :as worker-state]
+            [logseq.db.sqlite.util :as sqlite-util]
             [logseq.db.test.helper :as db-test]
-            [logseq.outliner.core :as outliner-core]))
+            [logseq.outliner.core :as outliner-core]
+            [promesa.core :as p]))
 
 (def ^:private test-repo "test-db-sync-repo")
 
@@ -63,6 +66,64 @@
             (is (= (:db/id child1') (:db/id (:block/parent parent'))))
             (is (= (:db/id page') (:db/id (:block/parent child1'))))))))))
 
+(deftest encrypt-decrypt-tx-data-test
+  (async done
+         (-> (p/let [aes-key (crypt/<generate-aes-key)
+                     tx-data [[:db/add 1 :block/title "hello"]
+                              [:db/add 2 :block/name "page"]
+                              [:db/add 3 :block/uuid (random-uuid)]]
+                     encrypted (#'db-sync/encrypt-tx-data aes-key tx-data)]
+               (is (not= tx-data encrypted))
+               (is (string? (nth (first encrypted) 3)))
+               (is (= (nth (second encrypted) 3)
+                      "page"))
+               (p/let [decrypted (#'db-sync/decrypt-tx-data aes-key encrypted)]
+                 (is (= tx-data decrypted))
+                 (done)))
+             (p/catch (fn [e]
+                        (is false (str e))
+                        (done))))))
+
+(deftest encrypt-decrypt-snapshot-rows-test
+  (async done
+         (-> (p/let [aes-key (crypt/<generate-aes-key)
+                     content (sqlite-util/write-transit-str {:db/id 1
+                                                             :block/title "hello"
+                                                             :block/name "page"})
+                     rows [[1 content nil]]
+                     encrypted (#'db-sync/encrypt-snapshot-rows aes-key rows)]
+               (is (not= rows encrypted))
+               (let [[_ content* _] (first encrypted)]
+                 (is (string? content*))
+                 (is (not= content content*)))
+               (p/let [decrypted (#'db-sync/decrypt-snapshot-rows aes-key encrypted)]
+                 (is (= rows decrypted))
+                 (done)))
+             (p/catch (fn [e]
+                        (is false (str e))
+                        (done))))))
+
+(deftest ensure-user-rsa-keys-test
+  (async done
+         (let [upload-called (atom nil)]
+           (with-redefs [db-sync/e2ee-base (fn [] "http://base")
+                         db-sync/<fetch-user-rsa-key-pair-raw (fn [_] (p/resolved {}))
+                         db-sync/<upload-user-rsa-key-pair! (fn [_ public-key encrypted-private-key]
+                                                              (reset! upload-called [public-key encrypted-private-key])
+                                                              (p/resolved {:public-key public-key
+                                                                           :encrypted-private-key encrypted-private-key}))
+                         crypt/<generate-rsa-key-pair (fn [] (p/resolved #js {:publicKey :pub :privateKey :priv}))
+                         crypt/<export-public-key (fn [_] (p/resolved :pub-export))
+                         crypt/<encrypt-private-key (fn [_ _] (p/resolved :priv-encrypted))
+                         worker-state/<invoke-main-thread (fn [_] (p/resolved {:password "pw"}))]
+             (-> (p/let [resp (db-sync/ensure-user-rsa-keys!)]
+                   (is (map? resp))
+                   (is (= 2 (count @upload-called)))
+                   (done))
+                 (p/catch (fn [e]
+                            (is false (str e))
+                            (done))))))))
+
 (deftest two-children-cycle-test
   (testing "cycle from remote sync overwrite client (2 children)"
     (let [{:keys [conn client-ops-conn child1 child2]} (setup-parent-child)]