add debug db tx-log and ws messages tables

src/main/frontend/handler/export.cljs

@@ -7,6 +7,7 @@
             [frontend.handler.assets :as assets-handler]
             [frontend.handler.export.common :as export-common-handler]
             [frontend.handler.notification :as notification]
+            [frontend.handler.user :as user-handler]
             [frontend.idb :as idb]
             [frontend.persist-db :as persist-db]
             [frontend.state :as state]
@@ -90,6 +91,24 @@
    (p/catch (fn [error]
               (js/console.error error)))))
 
+(defn export-repo-as-debug-log-sqlite!
+  [repo]
+  (if-not (and (state/get-auth-id-token) (user-handler/rtc-group?))
+    (notification/show! "Debug log export is limited to team members." :warning)
+    (->
+     (p/let [data (state/<invoke-db-worker-direct-pass :thread-api/export-debug-log-db repo)]
+       (if-not data
+         (notification/show! "Debug log db is not available for this graph." :warning)
+         (let [filename (file-name (str repo "_debug-log") "sqlite")
+               url (js/URL.createObjectURL (js/Blob. #js [data]))
+               anchor (.createElement js/document "a")]
+           (set! (.-href anchor) url)
+           (set! (.-download anchor) filename)
+           (.click anchor)
+           (js/URL.revokeObjectURL url))))
+     (p/catch (fn [error]
+                (js/console.error error))))))
+
 ;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Export to roam json ;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;

src/main/frontend/persist_db/browser.cljs

@@ -34,7 +34,8 @@
         (->> (m/watch state/state)
              (m/eduction
               (map #(select-keys % [:git/current-repo :config
-                                    :auth/id-token :auth/access-token :auth/refresh-token]))
+                                    :auth/id-token :auth/access-token :auth/refresh-token
+                                    :user/info]))
               (dedupe)))
         <init-sync-done? (p/deferred)
         task (m/reduce

src/main/frontend/worker/db_listener.cljs

@@ -4,6 +4,7 @@
             [datascript.core :as d]
             [frontend.common.thread-api :as thread-api]
             [frontend.worker.pipeline :as worker-pipeline]
+            [frontend.worker.rtc.debug-log :as rtc-debug-log]
             [frontend.worker.rtc.gen-client-op :as gen-client-op]
             [frontend.worker.search :as search]
             [frontend.worker.shared-service :as shared-service]
@@ -84,6 +85,7 @@
       (d/listen! conn ::listen-db-changes!
                  (fn listen-db-changes!-inner
                    [{:keys [tx-data _db-before _db-after tx-meta] :as tx-report}]
+                   (rtc-debug-log/log-tx! repo tx-data tx-meta)
                    (remove-old-embeddings-and-reset-new-updates! conn tx-data tx-meta)
 
                    (let [tx-meta (merge (batch-tx/get-batch-opts) tx-meta)

src/main/frontend/worker/db_worker.cljs

@@ -28,6 +28,7 @@
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.core :as rtc.core]
             [frontend.worker.rtc.db-listener]
+            [frontend.worker.rtc.debug-log :as rtc-debug-log]
             [frontend.worker.rtc.migrate :as rtc-migrate]
             [frontend.worker.search :as search]
             [frontend.worker.shared-service :as shared-service]
@@ -94,12 +95,15 @@
       nil)))
 
 (def repo-path "/db.sqlite")
+(def debug-log-path "/debug-log/db.sqlite")
 
 (defn- <export-db-file
-  [repo]
-  (p/let [^js pool (<get-opfs-pool repo)]
-    (when pool
-      (.exportFile ^js pool repo-path))))
+  ([repo]
+   (<export-db-file repo repo-path))
+  ([repo path]
+   (p/let [^js pool (<get-opfs-pool repo)]
+     (when pool
+       (.exportFile ^js pool path)))))
 
 (defn- <import-db
   [^js pool data]
@@ -153,27 +157,28 @@
       (restore-data-from-addr db addr))))
 
 (defn- close-db-aux!
-  [repo ^Object db ^Object search ^Object client-ops]
+  [repo ^Object db ^Object search ^Object client-ops ^Object debug-log]
   (swap! *sqlite-conns dissoc repo)
   (swap! *datascript-conns dissoc repo)
   (swap! *client-ops-conns dissoc repo)
   (when db (.close db))
   (when search (.close search))
   (when client-ops (.close client-ops))
+  (when debug-log (.close debug-log))
   (when-let [^js pool (worker-state/get-opfs-pool repo)]
     (.pauseVfs pool))
   (swap! *opfs-pools dissoc repo))
 
 (defn- close-other-dbs!
   [repo]
-  (doseq [[r {:keys [db search client-ops]}] @*sqlite-conns]
+  (doseq [[r {:keys [db search client-ops debug-log]}] @*sqlite-conns]
     (when-not (= repo r)
-      (close-db-aux! r db search client-ops))))
+      (close-db-aux! r db search client-ops debug-log))))
 
 (defn close-db!
   [repo]
-  (let [{:keys [db search client-ops]} (get @*sqlite-conns repo)]
-    (close-db-aux! repo db search client-ops)))
+  (let [{:keys [db search client-ops debug-log]} (get @*sqlite-conns repo)]
+    (close-db-aux! repo db search client-ops debug-log)))
 
 (defn reset-db!
   [repo db-transit-str]
@@ -198,8 +203,9 @@
                 (.unpauseVfs pool))
             db (new (.-OpfsSAHPoolDb pool) repo-path)
             search-db (new (.-OpfsSAHPoolDb pool) (str "search" repo-path))
-            client-ops-db (new (.-OpfsSAHPoolDb pool) (str "client-ops-" repo-path))]
-      [db search-db client-ops-db])))
+            client-ops-db (new (.-OpfsSAHPoolDb pool) (str "client-ops-" repo-path))
+            debug-log-db (new (.-OpfsSAHPoolDb pool) (str "debug-log" repo-path))]
+      [db search-db client-ops-db debug-log-db])))
 
 (defn- enable-sqlite-wal-mode!
   [^Object db]
@@ -224,18 +230,20 @@
 (defn- <create-or-open-db!
   [repo {:keys [config datoms] :as opts}]
   (when-not (worker-state/get-sqlite-conn repo)
-    (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
+    (p/let [[db search-db client-ops-db debug-log-db :as dbs] (get-dbs repo)
             storage (new-sqlite-storage db)
             client-ops-storage (when-not @*publishing?
                                  (new-sqlite-storage client-ops-db))
             db-based? true]
       (swap! *sqlite-conns assoc repo {:db db
                                        :search search-db
-                                       :client-ops client-ops-db})
+                                       :client-ops client-ops-db
+                                       :debug-log debug-log-db})
       (doseq [db' dbs]
         (enable-sqlite-wal-mode! db'))
       (common-sqlite/create-kvs-table! db)
       (when-not @*publishing? (common-sqlite/create-kvs-table! client-ops-db))
+      (rtc-debug-log/create-tables! debug-log-db)
       (search/create-tables-and-triggers! search-db)
       (ldb/register-transact-pipeline-fn!
        (fn [tx-report]
@@ -564,6 +572,18 @@
   (p/let [data (<export-db-file repo)]
     (Comlink/transfer data #js [(.-buffer data)])))
 
+(def-thread-api :thread-api/export-debug-log-db
+  [repo]
+  (when-let [^js db (worker-state/get-sqlite-conn repo :debug-log)]
+    (.exec db "PRAGMA wal_checkpoint(2)"))
+  (-> (p/let [data (<export-db-file
+                    repo
+                    debug-log-path)]
+        (when data
+          (Comlink/transfer data #js [(.-buffer data)])))
+      (p/catch (fn [error]
+                 (throw error)))))
+
 (def-thread-api :thread-api/import-db
   [repo data]
   (when-not (string/blank? repo)

src/main/frontend/worker/rtc/debug_log.cljs

@@ -0,0 +1,49 @@
+(ns frontend.worker.rtc.debug-log
+  "RTC debug logging stored in per-graph sqlite db."
+  (:require [frontend.worker.state :as worker-state]
+            [lambdaisland.glogi :as log]
+            [logseq.common.util :as common-util]))
+
+(defn create-tables!
+  [^js db]
+  (when db
+    (.exec db "CREATE TABLE IF NOT EXISTS tx_log (id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, tx_data TEXT, tx_meta TEXT)")
+    (.exec db "CREATE TABLE IF NOT EXISTS messages (id INTEGER PRIMARY KEY AUTOINCREMENT, created_at INTEGER NOT NULL, direction TEXT NOT NULL, message TEXT NOT NULL)")))
+
+(defn- enabled?
+  []
+  (worker-state/rtc-debug-enabled?))
+
+(defn- safe-str
+  [value]
+  (try
+    (pr-str value)
+    (catch :default _
+      (str value))))
+
+(defn- insert!
+  [^js db sql params]
+  (try
+    (.exec db #js {:sql sql
+                   :bind (clj->js params)})
+    (catch :default e
+      (log/error :rtc-debug-log-insert-failed e))))
+
+(defn log-tx!
+  [repo tx-data tx-meta]
+  (when repo
+    (when-let [db (worker-state/get-sqlite-conn repo :debug-log)]
+      (insert! db
+               "INSERT INTO tx_log (created_at, tx_data, tx_meta) VALUES (?1, ?2, ?3)"
+               [(common-util/time-ms) (safe-str tx-data) (safe-str tx-meta)])
+      (prn :debug :log-tx tx-meta))))
+
+(defn log-ws-message!
+  ([direction message]
+   (log-ws-message! (worker-state/get-current-repo) direction message))
+  ([repo direction message]
+   (when (and (enabled?) repo message)
+     (when-let [db (worker-state/get-sqlite-conn repo :debug-log)]
+       (insert! db
+                "INSERT INTO messages (created_at, direction, message) VALUES (?1, ?2, ?3)"
+                [(common-util/time-ms) (name direction) (str message)])))))

src/main/frontend/worker/rtc/ws.cljs

@@ -5,6 +5,7 @@
   (:require [cljs-http-missionary.client :as http]
             [frontend.common.missionary :as c.m]
             [frontend.worker.flows :as worker-flows]
+            [frontend.worker.rtc.debug-log :as rtc-debug-log]
             [frontend.worker.rtc.exception :as r.ex]
             [frontend.worker.rtc.malli-schema :as rtc-schema]
             [missionary.core :as m]))
@@ -137,6 +138,7 @@
   (m/sp
     (let [decoded-message (rtc-schema/data-to-ws-coercer message)
           message-str (js/JSON.stringify (clj->js (rtc-schema/data-to-ws-encoder decoded-message)))]
+      (rtc-debug-log/log-ws-message! :send message-str)
       (m/? ((:send mws) message-str)))))
 
 (defn- recv-flow*
@@ -144,6 +146,9 @@
   [m-ws]
   (assert (some? (:recv-flow m-ws)) m-ws)
   (m/eduction
+   (map (fn [message]
+          (rtc-debug-log/log-ws-message! :recv message)
+          message))
    (map #(js->clj (js/JSON.parse %) :keywordize-keys true))
    (map (fn [m]
           (if (contains?

src/main/frontend/worker/state.cljs

@@ -38,6 +38,8 @@
                        :auth/access-token nil
                        :auth/refresh-token nil
 
+                       :user/info nil
+
                        :rtc/downloading-graph? false
 
                        ;; thread atoms, these atoms' value are syncing from ui-thread
@@ -46,7 +48,7 @@
 (defonce *rtc-ws-url (atom nil))
 
 (defonce *sqlite (atom nil))
-;; repo -> {:db conn :search conn :client-ops conn}
+;; repo -> {:db conn :search conn :client-ops conn :debug-log conn}
 (defonce *sqlite-conns (atom {}))
 ;; repo -> conn
 (defonce *datascript-conns (atom nil))
@@ -61,7 +63,7 @@
 (defn get-sqlite-conn
   ([repo] (get-sqlite-conn repo :db))
   ([repo which-db]
-   (assert (contains? #{:db :search :client-ops} which-db) which-db)
+   (assert (contains? #{:db :search :client-ops :debug-log} which-db) which-db)
    (get-in @*sqlite-conns [repo which-db])))
 
 (defn get-datascript-conn
@@ -119,6 +121,15 @@
   []
   (:auth/id-token @*state))
 
+(defn get-user-groups
+  []
+  (set (get-in @*state [:user/info :UserGroups])))
+
+(defn rtc-debug-enabled?
+  []
+  (and (get-id-token)
+       (contains? (get-user-groups) "team")))
+
 (comment
   (defn mobile?
     []

src/main/logseq/api.cljs

@@ -83,6 +83,7 @@
 (def ^:export clear_right_sidebar_blocks api-app/clear_right_sidebar_blocks)
 (def ^:export push_state api-app/push_state)
 (def ^:export replace_state api-app/replace_state)
+(def ^:export export_debug_log_db api-app/export_debug_log_db)
 
 ;; db
 (def ^:export q api-db/q)

src/main/logseq/api/app.cljs

@@ -8,6 +8,7 @@
             [frontend.db.utils :as db-utils]
             [frontend.handler.command-palette :as palette-handler]
             [frontend.handler.config :as config-handler]
+            [frontend.handler.export :as export-handler]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.recent :as recent-handler]
             [frontend.handler.route :as route-handler]
@@ -172,3 +173,8 @@
       (if-let [page-name (and page? (:name params))]
         (route-handler/redirect-to-page! page-name {:anchor (:anchor query) :push false})
         (rfe/replace-state k params query)))))
+
+(def export_debug_log_db
+  (fn []
+    (when-let [repo (state/get-current-repo)]
+      (export-handler/export-repo-as-debug-log-sqlite! repo))))