Merge branch 'feat/db' into feat/repeated-tasks

.carve/ignore

@@ -80,11 +80,11 @@ frontend.ui/_emoji-init-data
 frontend.worker.rtc.op-mem-layer/_sync-loop-canceler
 ;; Used by shadow.cljs
 frontend.worker.db-worker/init
-;; WIP fn, remove when it's ready
-frontend.worker.rtc.asset-sync/<loop-for-assets-sync
 ;; Future use?
 frontend.worker.rtc.hash/hash-blocks
 ;; Repl fn
 frontend.rum/use-atom-in
 ;; missionary utils
-frontend.common.missionary-util/<!
+frontend.common.missionary/<!
+;; only used in deps/common
+frontend.common.missionary/background-task-running?

.lsp/config.edn

@@ -1,4 +1,13 @@
 {:source-aliases #{:cljs}
  :source-paths-ignore-regex ["src/resources" "target.*"]
  :paths-ignore-regex ["src/resources"]
- :clean {:ns-inner-blocks-indentation :same-line}}
+ :clean {:ns-inner-blocks-indentation :same-line}
+ :additional-snippets [{:name "profile"
+                        :detail "Insert profile-fn"
+                        :snippet
+                        "
+(comment
+(require '[logseq.common.profile :as c.p])
+(do (vreset! c.p/*key->call-count {})
+    (vreset! c.p/*key->time-sum {}))
+(c.p/profile-fn! $1) )"}]}

deps/common/.carve/ignore

@@ -2,3 +2,8 @@
 logseq.common.graph/get-files
 ;; API fn
 logseq.common.graph/read-directories
+
+;; Profile utils
+logseq.common.profile/profile-fn!
+logseq.common.profile/*key->call-count
+logseq.common.profile/*key->time-sum

deps/common/deps.edn

@@ -1,7 +1,9 @@
 {:paths ["src" "resources"]
  :deps {com.andrewmcveigh/cljs-time           {:git/url "https://github.com/logseq/cljs-time" ;; fork
                                                :sha     "5704fbf48d3478eedcf24d458c8964b3c2fd59a9"}
-        missionary/missionary                 {:mvn/version "b.39"}}
+        missionary/missionary                 {:mvn/version "b.39"}
+        org.clojure/core.async                {:mvn/version "1.6.673"}
+        funcool/promesa                       {:mvn/version "11.0.678"}}
  :aliases
  {:test {:extra-paths ["test"]
          :extra-deps  {olical/cljs-test-runner   {:mvn/version "3.8.0"}

deps/common/src/logseq/common/profile.clj

@@ -0,0 +1,18 @@
+(ns logseq.common.profile
+  "Utils for profiling")
+
+(defmacro profile-fn!
+  [f & {:keys [print-on-call? gen-k-fn]
+        :or {print-on-call? true}}]
+  `(let [origin-f# ~f
+         gen-k-fn# (or ~gen-k-fn (constantly (keyword ~f)))]
+     (set! ~f (fn [& args#]
+                (let [start# (cljs.core/system-time)
+                      r# (apply origin-f# args#)
+                      end# (cljs.core/system-time)
+                      k# (gen-k-fn# r#)]
+                  (vswap! *key->call-count update k# inc)
+                  (vswap! *key->time-sum update k# #(+ % (- end# start#)))
+                  (when ~print-on-call?
+                    (println "call-count:" (get @*key->call-count k#) "time-sum(ms):" (get @*key->time-sum k#)))
+                  r#)))))

deps/common/src/logseq/common/profile.cljs

@@ -0,0 +1,11 @@
+(ns logseq.common.profile
+  "Utils for profiling"
+  (:require-macros [logseq.common.profile]))
+
+(def *key->call-count
+  "key -> count"
+  (volatile! {}))
+
+(def *key->time-sum
+  "docstring"
+  (volatile! {}))

deps/db/src/logseq/db.cljs

@@ -3,22 +3,22 @@
    For shared file graph only fns, use logseq.graph-parser.db"
   (:require [clojure.set :as set]
             [clojure.string :as string]
+            [clojure.walk :as walk]
             [datascript.core :as d]
             [datascript.impl.entity :as de]
             [logseq.common.config :as common-config]
             [logseq.common.util :as common-util]
+            [logseq.common.util.namespace :as ns-util]
+            [logseq.common.util.page-ref :as page-ref]
             [logseq.common.uuid :as common-uuid]
             [logseq.db.frontend.class :as db-class]
             [logseq.db.frontend.delete-blocks :as delete-blocks] ;; Load entity extensions
-            [logseq.db.frontend.entity-plus]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
+            [logseq.db.frontend.property :as db-property]
             [logseq.db.frontend.rules :as rules]
             [logseq.db.sqlite.common-db :as sqlite-common-db]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.db.frontend.property :as db-property]
-            [logseq.common.util.namespace :as ns-util]
-            [logseq.common.util.page-ref :as page-ref]
-            [clojure.walk :as walk])
+            [logseq.db.sqlite.util :as sqlite-util])
   (:refer-clojure :exclude [object?]))
 
 (defonce *transact-fn (atom nil))
@@ -192,7 +192,7 @@
 
 (def get-first-page-by-name sqlite-common-db/get-first-page-by-name)
 
-(def db-based-graph? entity-util/db-based-graph?)
+(def db-based-graph? entity-plus/db-based-graph?)
 
 (defn page-exists?
   "Returns truthy value if page exists.

deps/db/src/logseq/db/frontend/content.cljs

@@ -4,7 +4,8 @@
             [logseq.common.util.page-ref :as page-ref]
             [datascript.core :as d]
             [logseq.common.util :as common-util]
-            [logseq.db.frontend.entity-util :as entity-util]))
+            [logseq.db.frontend.entity-util :as entity-util]
+            [logseq.db.frontend.entity-plus :as entity-plus]))
 
 ;; [[uuid]]
 (def id-ref-pattern
@@ -121,7 +122,7 @@
 (defn update-block-content
   "Replace `[[internal-id]]` with `[[page name]]`"
   [db item eid]
-  (if (entity-util/db-based-graph? db)
+  (if (entity-plus/db-based-graph? db)
     (if-let [content (:block/title item)]
       (let [refs (:block/refs (d/entity db eid))]
         (assoc item :block/title (id-ref->title-ref content refs false)))

deps/db/src/logseq/db/frontend/delete_blocks.cljs

@@ -5,7 +5,8 @@
             [logseq.common.util.page-ref :as page-ref]
             [datascript.core :as d]
             [clojure.string :as string]
-            [logseq.db.frontend.entity-util :as entity-util]))
+            [logseq.db.frontend.entity-util :as entity-util]
+            [logseq.db.frontend.entity-plus :as entity-plus]))
 
 (defn- replace-ref-with-deleted-block-title
   [block ref-raw-title]
@@ -57,7 +58,7 @@
     (when (seq retracted-block-ids)
       (let [retracted-blocks (map #(d/entity db %) retracted-block-ids)
             retracted-tx (build-retracted-tx retracted-blocks)
-            macros-tx (when-not (entity-util/db-based-graph? db)
+            macros-tx (when-not (entity-plus/db-based-graph? db)
                         (mapcat (fn [b]
                                   ;; Only delete if last reference
                                   (keep #(when (<= (count (:block/_macros (d/entity db (:db/id %))))

deps/db/src/logseq/db/frontend/entity_plus.cljc

@@ -5,23 +5,76 @@
   ;; Disable clj linters since we don't support clj
   #?(:clj {:clj-kondo/config {:linters {:unresolved-namespace {:level :off}
                                         :unresolved-symbol {:level :off}}}})
-  (:require [cljs.core]
-            #?(:org.babashka/nbb [datascript.db])
+  (:require #?(:org.babashka/nbb [datascript.db])
+            [cljs.core]
+            [datascript.core :as d]
             [datascript.impl.entity :as entity :refer [Entity]]
-            [logseq.db.frontend.content :as db-content]
-            [logseq.db.frontend.property :as db-property]
-            [logseq.db.frontend.entity-util :as entity-util]
             [logseq.common.util.date-time :as date-time-util]
-            [datascript.core :as d]))
-
-(def db-based-graph? entity-util/db-based-graph?)
-
-(def lookup-entity @#'entity/lookup-entity)
+            [logseq.db.frontend.entity-util :as entity-util]
+            [logseq.db.frontend.property :as db-property]))
+
+(def immutable-db-idents
+  "These db-ident entities are immutable,
+  it means `(db/entity :block/title)` always return same result"
+  #{:block/created-at :block/updated-at
+    :block/uuid :block/title :block/tags :block/name :block/format
+    :block/schema :block/path-refs :block/refs :block/tx-id :block/type
+    :block/page :block/parent :block/order :block/journal-day :block/closed-value-property
+    :block/link :block/marker :block/warning :block/collapsed? :block/deadline :block/scheduled :block/level
+    :block/pre-block? :block/heading-level
+
+    :block/_refs :logseq.property/_query
+
+    :block.temp/search? :block.temp/ast-title :block.temp/ast-body
+    :block.temp/fully-loaded? :block.temp/top? :block.temp/bottom?
+
+    :db/cardinality :db/ident :db/index :db/valueType
+
+    :logseq.kv/db-type
+
+    :logseq.property.node/display-type :logseq.property/icon
+    :logseq.property.asset/type :logseq.property.asset/checksum
+    :logseq.property/created-from-property
+
+    :logseq.class/Query :logseq.class/Journal :logseq.class/Cards :logseq.class/Task})
+
+(def ^:private lookup-entity @#'entity/lookup-entity)
+
+(def ^:private *seen-immutable-entities (volatile! {}))
+
+(defn reset-immutable-entities-cache!
+  []
+  (vreset! *seen-immutable-entities {}))
+
+(def ^:private *reset-cache-background-task-running?
+  ;; missionary is not compatible with nbb, so entity-memoized is disabled in nbb
+  (delay
+    #?(:org.babashka/nbb false
+       :cljs (when-let [f (resolve 'frontend.common.missionary/background-task-running?)]
+               (f :logseq.db.frontend.entity-plus/reset-immutable-entities-cache!)))))
+
+(defn entity-memoized
+  [db eid]
+  (if (and @*reset-cache-background-task-running?
+           (qualified-keyword? eid)
+           (contains? immutable-db-idents eid))
+    (if-let [e (find @*seen-immutable-entities eid)]
+      (val e)
+      (let [r (d/entity db eid)]
+        (when r (vswap! *seen-immutable-entities assoc eid r))
+        r))
+    (d/entity db eid)))
+
+(defn db-based-graph?
+  "Whether the current graph is db-only"
+  [db]
+  (when db
+    (= "db" (:kv/value (entity-memoized db :logseq.kv/db-type)))))
 
 (defn- get-journal-title
   [db e]
   (date-time-util/int->journal-title (:block/journal-day e)
-                                     (:logseq.property.journal/title-format (d/entity db :logseq.class/Journal))))
+                                     (:logseq.property.journal/title-format (entity-memoized db :logseq.class/Journal))))
 
 (defn- get-block-title
   [^Entity e k default-value]
@@ -36,7 +89,7 @@
          (let [result (lookup-entity e k default-value)
                refs (:block/refs e)
                result' (if (and (string? result) refs)
-                         (db-content/id-ref->title-ref result refs search?)
+                         ((resolve 'logseq.db.frontend.content/id-ref->title-ref) result refs search?)
                          result)]
            (or result' default-value)))))))
 
@@ -51,7 +104,7 @@
        result
        ;; property default value
        (when (qualified-keyword? k)
-         (when-let [property (d/entity db k)]
+         (when-let [property (entity-memoized db k)]
            (let [schema (lookup-entity property :block/schema nil)]
              (if (= :checkbox (:type schema))
                (lookup-entity property :logseq.property/scalar-default-value nil)
@@ -107,7 +160,9 @@
   [^js this]
   (let [v @(.-cache this)
         v' (if (:block/title v)
-             (assoc v :block/title (db-content/id-ref->title-ref (:block/title v) (:block/refs this) (:block.temp/search? this)))
+             (assoc v :block/title
+                    ((resolve 'logseq.db.frontend.content/id-ref->title-ref)
+                     (:block/title v) (:block/refs this) (:block.temp/search? this)))
              v)]
     (concat (seq v')
             (seq (.-kv this)))))

deps/db/src/logseq/db/frontend/entity_util.cljs

@@ -1,22 +1,22 @@
 (ns logseq.db.frontend.entity-util
   "Lower level entity util fns used across db namespaces"
-  (:require [datascript.core :as d]
-            [clojure.string :as string]
+  (:require [clojure.string :as string]
+            [datascript.db]
             [datascript.impl.entity :as de])
   (:refer-clojure :exclude [object?]))
 
-(defn db-based-graph?
-  "Whether the current graph is db-only"
-  [db]
-  (when db
-    (= "db" (:kv/value (d/entity db :logseq.kv/db-type)))))
-
 (defn- has-tag?
   [entity tag-ident]
-  (let [tags (:block/tags entity)]
-    (some (fn [t] (or (= (:db/ident t) tag-ident)
-                      (= t tag-ident)))
-          (if (coll? tags) tags [tags]))))
+  (some (fn [t]
+          (or (keyword-identical? (:db/ident t) tag-ident)
+              (keyword-identical? t tag-ident)))
+        (:block/tags entity)))
+
+(comment
+  (require '[logseq.common.profile :as c.p])
+  (do (vreset! c.p/*key->call-count {})
+      (vreset! c.p/*key->time-sum {}))
+  (c.p/profile-fn! has-tag? :print-on-call? false))
 
 (defn internal-page?
   [entity]
@@ -24,8 +24,8 @@
 
 (defn class?
   [entity]
-  (or (= (:db/ident entity) :logseq.class/Tag)
-      (has-tag? entity :logseq.class/Tag)))
+  (or (has-tag? entity :logseq.class/Tag)
+      (keyword-identical? (:db/ident entity) :logseq.class/Tag)))
 
 (defn property?
   [entity]
@@ -38,7 +38,7 @@
    ;; db based graph
    (has-tag? entity :logseq.class/Whiteboard)
    ;; file based graph
-   (= "whiteboard" (:block/type entity))))
+   (identical? "whiteboard" (:block/type entity))))
 
 (defn closed-value?
   [entity]
@@ -51,7 +51,7 @@
    ;; db based graph
    (has-tag? entity :logseq.class/Journal)
    ;; file based graph
-   (= "journal" (:block/type entity))))
+   (identical? "journal" (:block/type entity))))
 
 (defn page?
   [entity]
@@ -92,4 +92,4 @@
                      :logseq.class/Journal :journal
                      :logseq.class/Whiteboard :whiteboard
                      :logseq.class/Page :page}]
-    (set (map #(ident->type (:db/ident %)) (:block/tags entity)))))
+    (set (map #(ident->type (:db/ident %)) (:block/tags entity)))))

deps/db/src/logseq/db/sqlite/common_db.cljs

@@ -1,15 +1,16 @@
 (ns logseq.db.sqlite.common-db
   "Common sqlite db fns for browser and node"
-  (:require [datascript.core :as d]
-            ["path" :as node-path]
+  (:require ["path" :as node-path]
+            [clojure.set :as set]
             [clojure.string :as string]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [logseq.common.util.date-time :as date-time-util]
-            [logseq.common.util :as common-util]
+            [datascript.core :as d]
             [logseq.common.config :as common-config]
+            [logseq.common.util :as common-util]
+            [logseq.common.util.date-time :as date-time-util]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.entity-util :as entity-util]
-            [clojure.set :as set]
-            [logseq.db.frontend.order :as db-order]))
+            [logseq.db.frontend.order :as db-order]
+            [logseq.db.sqlite.util :as sqlite-util]))
 
 (defn- get-pages-by-name
   [db page-name]
@@ -81,7 +82,7 @@
 
 (defn- property-with-values
   [db block]
-  (when (entity-util/db-based-graph? db)
+  (when (entity-plus/db-based-graph? db)
     (let [block (d/entity db (:db/id block))]
       (->> (:block/properties block)
            vals
@@ -252,7 +253,7 @@
   "Returns current database schema and initial data.
    NOTE: This fn is called by DB and file graphs"
   [db]
-  (let [db-graph? (entity-util/db-based-graph? db)
+  (let [db-graph? (entity-plus/db-based-graph? db)
         _ (when db-graph?
             (reset! db-order/*max-key (db-order/get-max-order db)))
         schema (:schema db)

deps/db/src/logseq/db/test/helper.cljs

@@ -1,9 +1,10 @@
 (ns ^:node-only logseq.db.test.helper
   "Main ns for providing test fns for DB graphs"
   (:require [datascript.core :as d]
+            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.frontend.schema :as db-schema]
             [logseq.db.sqlite.build :as sqlite-build]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
-            [logseq.db.frontend.schema :as db-schema]))
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]))
 
 (defn find-block-by-content
   "Find first block by exact block string or by fuzzier regex"
@@ -43,6 +44,7 @@
   []
   (let [conn (d/create-conn db-schema/schema-for-db-based-graph)
         _ (d/transact! conn (sqlite-create-graph/build-db-initial-data "{}"))]
+    (entity-plus/reset-immutable-entities-cache!)
     conn))
 
 (defn create-conn-with-blocks
@@ -50,4 +52,4 @@
   [opts]
   (let [conn (create-conn)
         _ (sqlite-build/create-blocks conn opts)]
-    conn))
+    conn))

deps/outliner/src/logseq/outliner/property.cljs

@@ -460,7 +460,7 @@
 
 (defn- property-with-position?
   [db property-id block position]
-  (let [property (d/entity db property-id)
+  (let [property (entity-plus/entity-memoized db property-id)
         schema (:block/schema property)]
     (and
      (= (:position schema) position)

deps/outliner/test/logseq/outliner/validate_test.cljs

@@ -1,8 +1,9 @@
 (ns logseq.outliner.validate-test
-  (:require [cljs.test :refer [deftest is are testing]]
+  (:require [cljs.test :refer [are deftest is testing]]
             [datascript.core :as d]
-            [logseq.outliner.validate :as outliner-validate]
-            [logseq.db.test.helper :as db-test]))
+            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.test.helper :as db-test]
+            [logseq.outliner.validate :as outliner-validate]))
 
 (deftest validate-block-title-unique-for-properties
   (let [conn (db-test/create-conn-with-blocks
@@ -97,10 +98,10 @@
 
     (testing "built-in tag can't have parent changed"
       (is (thrown-with-msg?
-            js/Error
-            #"Can't change.*built-in"
-            (outliner-validate/validate-parent-property (d/entity @conn :logseq.class/Task)
-                                                        [(d/entity @conn :logseq.class/Cards)]))))))
+           js/Error
+           #"Can't change.*built-in"
+           (outliner-validate/validate-parent-property (entity-plus/entity-memoized @conn :logseq.class/Task)
+                                                       [(entity-plus/entity-memoized @conn :logseq.class/Cards)]))))))
 
 (deftest validate-tags-property
   (let [conn (db-test/create-conn-with-blocks

deps/publishing/src/logseq/publishing/db.cljs

@@ -1,17 +1,17 @@
 (ns logseq.publishing.db
   "Provides db fns and associated util fns for publishing"
-  (:require [datascript.core :as d]
-            [logseq.db.frontend.rules :as rules]
-            [clojure.set :as set]
+  (:require [clojure.set :as set]
             [clojure.string :as string]
-            [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.malli-schema :as db-malli-schema]))
+            [datascript.core :as d]
+            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.frontend.malli-schema :as db-malli-schema]
+            [logseq.db.frontend.rules :as rules]))
 
 (defn ^:api get-area-block-asset-url
   "Returns asset url for an area block used by pdf assets. This lives in this ns
   because it is used by this dep and needs to be independent from the frontend app"
   [db block page]
-  (let [db-based? (entity-util/db-based-graph? db)]
+  (let [db-based? (entity-plus/db-based-graph? db)]
     (when-some [uuid' (:block/uuid block)]
       (if db-based?
         (when-let [image (:logseq.property.pdf/hl-image block)]
@@ -105,7 +105,7 @@
 
 (defn- hl-type-area-fn
   [db]
-  (if (entity-util/db-based-graph? db)
+  (if (entity-plus/db-based-graph? db)
     (fn [datom]
       (and (= :logseq.property.pdf/hl-type (:a datom))
            (= (keyword (:v datom)) :area)))

src/main/frontend/background_tasks.cljs

@@ -0,0 +1,15 @@
+(ns frontend.background-tasks
+  "Some background tasks"
+  (:require [frontend.flows :as flows]
+            [frontend.common.missionary :as c.m]
+            [logseq.db.frontend.entity-plus :as entity-plus]
+            [missionary.core :as m]))
+
+(c.m/run-background-task
+ :logseq.db.frontend.entity-plus/reset-immutable-entities-cache!
+ (m/reduce
+  (fn [_ repo]
+    (when (some? repo)
+      (prn :reset-immutable-entities-cache!)
+      (entity-plus/reset-immutable-entities-cache!)))
+  flows/current-repo-flow))

src/main/frontend/common/missionary_util.clj → src/main/frontend/common/missionary.clj

@@ -1,4 +1,4 @@
-(ns frontend.common.missionary-util
+(ns frontend.common.missionary
   "Macros for missionary"
   (:require [missionary.core :as m]))
 

src/main/frontend/common/missionary_util.cljs → src/main/frontend/common/missionary.cljs

@@ -1,12 +1,10 @@
-(ns frontend.common.missionary-util
+(ns frontend.common.missionary
   "Utils based on missionary. Used by frontend and worker namespaces"
-  (:require-macros [frontend.common.missionary-util])
+  (:require-macros [frontend.common.missionary])
   (:require [cljs.core.async.impl.channels]
             [clojure.core.async :as a]
             [missionary.core :as m]
-            [promesa.protocols :as pt])
-  ;; (:import [missionary Cancelled])
-  )
+            [promesa.protocols :as pt]))
 
 (defn continue-flow
   "ensure f is a continuous flow"
@@ -95,16 +93,20 @@
 
 (defn run-background-task
   "Run task.
-  will cancel last same key background-task,
-  useful when developing (to avoid: reload cljs then run multiple same tasks)"
-  [key task]
-  (when-let [canceler (get @*background-task-cancelers key)]
+  Cancel last same key background-task if exists(to avoid: reload cljs then run multiple same tasks)"
+  [key' task]
+  (when-let [canceler (get @*background-task-cancelers key')]
     (canceler)
-    (vswap! *background-task-cancelers assoc key nil))
-  (let [canceler (run-task task key)]
-    (vswap! *background-task-cancelers assoc key canceler)
+    (vswap! *background-task-cancelers assoc key' nil))
+  (prn :run-background-task key')
+  (let [canceler (run-task task key')]
+    (vswap! *background-task-cancelers assoc key' canceler)
     nil))
 
+(defn background-task-running?
+  [key']
+  (contains? @*background-task-cancelers key'))
+
 (comment
   (defn >!
     "Return a task that

src/main/frontend/components/block.cljs

@@ -19,8 +19,8 @@
             [frontend.components.property.value :as pv]
             [frontend.components.query :as query]
             [frontend.components.query.builder :as query-builder-component]
-            [frontend.components.svg :as svg]
             [frontend.components.select :as select]
+            [frontend.components.svg :as svg]
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
             [frontend.date :as date]
@@ -78,6 +78,7 @@
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
             [logseq.db.frontend.content :as db-content]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.text :as text]
@@ -2285,7 +2286,9 @@
          ;; highlight ref block (area)
          (when area? [(hl-ref)])
 
-         (when (and (seq block-ast-title) (ldb/class-instance? (db/entity :logseq.class/Cards) block))
+         (when (and (seq block-ast-title) (ldb/class-instance?
+                                           (entity-plus/entity-memoized (db/get-db) :logseq.class/Cards)
+                                           block))
            [(ui/tooltip
              (shui/button
               {:variant :ghost
@@ -2302,7 +2305,8 @@
   (let [collapsed? (:collapsed? config)
         block' (db/entity (:db/id block))
         node-display-type (:logseq.property.node/display-type block')
-        query? (ldb/class-instance? (db/entity :logseq.class/Query) block')
+        db (db/get-db)
+        query? (ldb/class-instance? (entity-plus/entity-memoized db :logseq.class/Query) block')
         query (:logseq.property/query block')
         advanced-query? (and query? (= :code node-display-type))]
     (cond
@@ -3277,7 +3281,8 @@
 (rum/defc query-property-cp < rum/reactive db-mixins/query
   [block config collapsed?]
   (let [block (db/entity (:db/id block))
-        query? (ldb/class-instance? (db/entity :logseq.class/Query) block)]
+        db (db/get-db)
+        query? (ldb/class-instance? (entity-plus/entity-memoized db :logseq.class/Query) block)]
     (when (and query? (not collapsed?))
       (let [query-id (:db/id (:logseq.property/query block))
             query (some-> query-id db/sub-block)
@@ -3496,7 +3501,7 @@
         (db-properties-cp config block {:in-block-container? true})])
 
      (when (and db-based? (not collapsed?) (not (or table? property?))
-                (ldb/class-instance? (db/entity :logseq.class/Query) block))
+                (ldb/class-instance? (entity-plus/entity-memoized (db/get-db) :logseq.class/Query) block))
        (let [query-block (:logseq.property/query (db/entity (:db/id block)))
              query-block (if query-block (db/sub-block (:db/id query-block)) query-block)
              query (:block/title query-block)
@@ -3505,7 +3510,9 @@
          [:div {:style {:padding-left 42}}
           (query/custom-query (wrap-query-components (assoc config
                                                             :dsl-query? (not advanced-query?)
-                                                            :cards? (ldb/class-instance? (db/entity :logseq.class/Cards) block)))
+                                                            :cards? (ldb/class-instance? (entity-plus/entity-memoized
+                                                                                          (db/get-db)
+                                                                                          :logseq.class/Cards) block)))
                               (if advanced-query? result {:builder nil
                                                           :query (query-builder-component/sanitize-q query)}))]))
 

src/main/frontend/components/container.cljs

@@ -1056,7 +1056,7 @@
       [:a#download.hidden]
       [:a#download-as-edn-v2.hidden]
       [:a#download-as-json-v2.hidden]
-      [:a#download-as-json-debug.hidden]
+      [:a#download-as-transit-debug.hidden]
       [:a#download-as-sqlite-db.hidden]
       [:a#download-as-roam-json.hidden]
       [:a#download-as-html.hidden]

src/main/frontend/components/export.cljs

@@ -94,9 +94,9 @@
             (t :export-zip)]])
         (when db-based?
           [:div
-           [:a.font-medium {:on-click #(export/export-repo-as-debug-json! current-repo)}
-            "Export debug JSON"]
-           [:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported json file, you can send it to us for debugging."]])
+           [:a.font-medium {:on-click #(export/export-repo-as-debug-transit! current-repo)}
+            "Export debug transit file"]
+           [:p.text-sm.opacity-70 "Any sensitive data will be removed in the exported transit file, you can send it to us for debugging."]])
 
         (when (util/electron?)
           [:div

src/main/frontend/components/header.cljs

@@ -4,13 +4,11 @@
             [cljs-time.core :as t]
             [clojure.string :as string]
             [dommy.core :as d]
-            [frontend.common.missionary-util :as c.m]
             [frontend.components.export :as export]
             [frontend.components.file-sync :as fs-sync]
             [frontend.components.page-menu :as page-menu]
             [frontend.components.plugins :as plugins]
             [frontend.components.right-sidebar :as sidebar]
-            [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.components.rtc.indicator :as rtc-indicator]
             [frontend.components.server :as server]
             [frontend.components.settings :as settings]
@@ -19,6 +17,7 @@
             [frontend.context.i18n :refer [t]]
             [frontend.db :as db]
             [frontend.handler :as handler]
+            [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.handler.page :as page-handler]
             [frontend.handler.plugin :as plugin-handler]
             [frontend.handler.route :as route-handler]
@@ -29,6 +28,7 @@
             [frontend.ui :as ui]
             [frontend.util :as util]
             [frontend.version :refer [version]]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [logseq.shui.ui :as shui]
             [logseq.shui.util :as shui-util]

src/main/frontend/components/imports.cljs

@@ -69,7 +69,7 @@
                           :error))))
 
 (defn- lsq-import-handler
-  [e & {:keys [sqlite? graph-name]}]
+  [e & {:keys [sqlite? debug-transit? graph-name]}]
   (let [file      (first (array-seq (.-files (.-target e))))
         file-name (some-> (gobj/get file "name")
                           (string/lower-case))
@@ -98,6 +98,31 @@
                                        (js/console.error e)))
             (.readAsArrayBuffer reader file))))
 
+      debug-transit?
+      (let [graph-name (string/trim graph-name)]
+        (cond
+          (string/blank? graph-name)
+          (notification/show! "Empty graph name." :error)
+
+          (repo-handler/graph-already-exists? graph-name)
+          (notification/show! "Please specify another name as another graph with this name already exists!" :error)
+
+          :else
+          (do
+            (state/set-state! :graph/importing :logseq)
+            (let [reader (js/FileReader.)
+                  import-f import-handler/import-from-debug-transit!]
+              (set! (.-onload reader)
+                    (fn [e]
+                      (let [text (.. e -target -result)]
+                        (import-f
+                         graph-name
+                         text
+                         #(do
+                            (state/set-state! :graph/importing nil)
+                            (finished-cb))))))
+              (.readAsText reader file)))))
+
       (or edn? json?)
       (do
         (state/set-state! :graph/importing :logseq)
@@ -142,11 +167,11 @@
 (rum/defcs set-graph-name-dialog
   < rum/reactive
   (rum/local "" ::input)
-  [state sqlite-input-e opts]
+  [state input-e opts]
   (let [*input (::input state)
         on-submit #(if (repo/invalid-graph-name? @*input)
                      (repo/invalid-graph-name-warning)
-                     (lsq-import-handler sqlite-input-e (assoc opts :graph-name @*input)))]
+                     (lsq-import-handler input-e (assoc opts :graph-name @*input)))]
     [:div.container
      [:div.sm:flex.sm:items-start
       [:div.mt-3.text-center.sm:mt-0.sm:text-left
@@ -184,74 +209,74 @@
                             ;; (js/console.log "[form] submit: " e (js->clj e))
                             (on-submit-fn (js->clj e :keywordize-keys true))
                             (shui/dialog-close!)))
-        [convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]
+         [convert-all-tags-input set-convert-all-tags-input!] (rum/use-state true)]
 
      (shui/form-provider form-ctx
-       [:form
-        {:on-submit on-submit-valid}
-
-        (shui/form-field {:name "graph-name"}
-                         (fn [field error]
-                           (shui/form-item
-                            (shui/form-label "New graph name")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "Graph name"} field)))
-                            (when error
-                              (shui/form-description
-                               [:b.text-red-800 (:message error)])))))
-
-        (shui/form-field {:name "convert-all-tags?"}
-                         (fn [field]
-                           (shui/form-item
-                            {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
-                            (shui/form-label "Import all tags")
-                            (shui/form-control
-                             (shui/checkbox {:checked (:value field)
-                                             :on-checked-change (fn [e]
-                                                                  ((:onChange field) e)
-                                                                  (set-convert-all-tags-input! (not convert-all-tags-input)))})))))
-
-        (shui/form-field {:name "tag-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import specific tags")
-                            (shui/form-control
-                             (shui/input (merge field
-                                                {:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
-                            (shui/form-description "Tags are case insensitive"))))
-
-        (shui/form-field {:name "remove-inline-tags?"}
-                         (fn [field]
-                           (shui/form-item
-                            {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
-                            (shui/form-label "Remove inline tags")
-                            (shui/form-description "Default behavior for DB graphs")
-                            (shui/form-control
-                             (shui/checkbox {:checked (:value field)
-                                             :on-checked-change (:onChange field)})))))
-
-        (shui/form-field {:name "property-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import additional tags from property values")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "e.g. type"} field)))
-                            (shui/form-description
-                             "Properties are case insensitive and separated by commas"))))
-
-        (shui/form-field {:name "property-parent-classes"}
-                         (fn [field _error]
-                           (shui/form-item
-                            {:class "pt-3"}
-                            (shui/form-label "Import tag parents from property values")
-                            (shui/form-control
-                             (shui/input (merge {:placeholder "e.g. parent"} field)))
-                            (shui/form-description
-                             "Properties are case insensitive and separated by commas"))))
-
-        (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
+                         [:form
+                          {:on-submit on-submit-valid}
+
+                          (shui/form-field {:name "graph-name"}
+                                           (fn [field error]
+                                             (shui/form-item
+                                              (shui/form-label "New graph name")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "Graph name"} field)))
+                                              (when error
+                                                (shui/form-description
+                                                 [:b.text-red-800 (:message error)])))))
+
+                          (shui/form-field {:name "convert-all-tags?"}
+                                           (fn [field]
+                                             (shui/form-item
+                                              {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
+                                              (shui/form-label "Import all tags")
+                                              (shui/form-control
+                                               (shui/checkbox {:checked (:value field)
+                                                               :on-checked-change (fn [e]
+                                                                                    ((:onChange field) e)
+                                                                                    (set-convert-all-tags-input! (not convert-all-tags-input)))})))))
+
+                          (shui/form-field {:name "tag-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import specific tags")
+                                              (shui/form-control
+                                               (shui/input (merge field
+                                                                  {:placeholder "tag 1, tag 2" :disabled convert-all-tags-input})))
+                                              (shui/form-description "Tags are case insensitive"))))
+
+                          (shui/form-field {:name "remove-inline-tags?"}
+                                           (fn [field]
+                                             (shui/form-item
+                                              {:class "pt-3 flex justify-start items-center space-x-3 space-y-0 my-3 pr-3"}
+                                              (shui/form-label "Remove inline tags")
+                                              (shui/form-description "Default behavior for DB graphs")
+                                              (shui/form-control
+                                               (shui/checkbox {:checked (:value field)
+                                                               :on-checked-change (:onChange field)})))))
+
+                          (shui/form-field {:name "property-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import additional tags from property values")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "e.g. type"} field)))
+                                              (shui/form-description
+                                               "Properties are case insensitive and separated by commas"))))
+
+                          (shui/form-field {:name "property-parent-classes"}
+                                           (fn [field _error]
+                                             (shui/form-item
+                                              {:class "pt-3"}
+                                              (shui/form-label "Import tag parents from property values")
+                                              (shui/form-control
+                                               (shui/input (merge {:placeholder "e.g. parent"} field)))
+                                              (shui/form-description
+                                               "Properties are case insensitive and separated by commas"))))
+
+                          (shui/button {:type "submit" :class "right-0 mt-3"} "Submit")]))])
 
 (defn- counts-from-entities
   [entities]
@@ -416,14 +441,14 @@
 (rum/defc import-indicator
   [importing?]
   (rum/use-effect!
-    (fn []
-      (when (and importing? (not (shui-dialog/get-modal :import-indicator)))
-        (shui/dialog-open! indicator-progress
-          {:id :import-indicator
-           :content-props
-           {:onPointerDownOutside #(.preventDefault %)
-            :onOpenAutoFocus #(.preventDefault %)}})))
-    [importing?])
+   (fn []
+     (when (and importing? (not (shui-dialog/get-modal :import-indicator)))
+       (shui/dialog-open! indicator-progress
+                          {:id :import-indicator
+                           :content-props
+                           {:onPointerDownOutside #(.preventDefault %)
+                            :onOpenAutoFocus #(.preventDefault %)}})))
+   [importing?])
   [:<>])
 
 (rum/defc importer < rum/reactive
@@ -468,6 +493,21 @@
                                       (import-file-to-db-handler e {}))
                                     1000)}]])
 
+          (when (or (util/electron?) util/web-platform?)
+            [:label.action-input.flex.items-center.mx-2.my-2
+             [:span.as-flex-center [:i (svg/logo 28)]]
+             [:span.flex.flex-col
+              [[:strong "Debug Transit"]
+               [:small "Import debug transit file into a new DB graph"]]]
+             ;; Test form style changes
+             #_[:a.button {:on-click #(import-file-to-db-handler nil {:import-graph-fn js/alert})} "Open"]
+             [:input.absolute.hidden
+              {:id "import-debug-transit"
+               :type "file"
+               :on-change (fn [e]
+                            (shui/dialog-open!
+                             #(set-graph-name-dialog e {:debug-transit? true})))}]])
+
           (when (and (util/electron?) support-file-based?)
             [:label.action-input.flex.items-center.mx-2.my-2
              [:span.as-flex-center [:i (svg/logo 28)]]

src/main/frontend/components/property.cljs

@@ -9,16 +9,16 @@
             [frontend.components.property.value :as pv]
             [frontend.components.select :as select]
             [frontend.components.svg :as svg]
-            [frontend.handler.property.util :as pu]
             [frontend.config :as config]
             [frontend.db :as db]
-            [frontend.db.model :as db-model]
             [frontend.db-mixins :as db-mixins]
             [frontend.db.async :as db-async]
+            [frontend.db.model :as db-model]
             [frontend.handler.db-based.property :as db-property-handler]
+            [frontend.handler.editor :as editor-handler]
             [frontend.handler.notification :as notification]
+            [frontend.handler.property.util :as pu]
             [frontend.handler.route :as route-handler]
-            [frontend.handler.editor :as editor-handler]
             [frontend.mixins :as mixins]
             [frontend.modules.shortcut.core :as shortcut]
             [frontend.state :as state]

src/main/frontend/components/rtc/indicator.cljs

@@ -1,16 +1,16 @@
 (ns frontend.components.rtc.indicator
   "RTC state indicator"
   (:require [cljs-time.core :as t]
-            [frontend.common.missionary-util :as c.m]
+            [clojure.pprint :as pprint]
+            [frontend.db :as db]
             [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.common.missionary :as c.m]
             [logseq.shui.ui :as shui]
             [missionary.core :as m]
-            [rum.core :as rum]
-            [clojure.pprint :as pprint]
-            [frontend.db :as db]))
+            [rum.core :as rum]))
 
 (comment
   (def rtc-state-schema

src/main/frontend/core.cljs

@@ -1,7 +1,9 @@
 (ns frontend.core
   "Entry ns for the mobile, browser and electron frontend apps"
   {:dev/always true}
-  (:require [frontend.common-keywords]
+  (:require [frontend.background-tasks]
+            [frontend.common-keywords]
+            [frontend.common.schema-register :as sr]
             [frontend.components.plugins :as plugins]
             [frontend.config :as config]
             [frontend.fs.sync :as sync]
@@ -11,7 +13,6 @@
             [frontend.log]
             [frontend.page :as page]
             [frontend.routes :as routes]
-            [frontend.common.schema-register :as sr]
             [frontend.spec]
             [logseq.api]
             [malli.dev.cljs :as md]
@@ -58,11 +59,28 @@
     (when config/dev?
       (js/setTimeout #(sync/<sync-start) 1000))))
 
+(comment
+  (def d-entity-count (volatile! 0))
+  (def ident->count (volatile! {}))
+  (def time-sum (volatile! 0))
+  (defn- setup-entity-profile!
+    []
+    (let [origin-d-entity d/entity]
+      (set! d/entity (fn [& args]
+                       (let [{r :result time :time} (util/with-time (apply origin-d-entity args))
+                             k (last args)]
+                         (vswap! d-entity-count inc)
+                         (vswap! ident->count update k inc)
+                         (vswap! time-sum #(+ time %))
+                         (println @d-entity-count (:db/id r) k (get @ident->count k) @time-sum "ms")
+                         r))))))
+
 (defn ^:export init []
   ;; init is called ONCE when the page loads
   ;; this is called in the index.html and must be exported
   ;; so it is available even in :advanced release builds
 
+  ;; (setup-entity-profile!)
   (plugin-handler/setup!
    #(handler/start! start)))
 

src/main/frontend/db/restore.cljs

@@ -10,10 +10,10 @@
 
 (defn restore-graph!
   "Restore db from SQLite"
-  [repo]
+  [repo & {:as opts}]
   (state/set-state! :graph/loading? true)
   (p/let [start-time (t/now)
-          data (persist-db/<fetch-init-data repo)
+          data (persist-db/<fetch-init-data repo opts)
           _ (assert (some? data) "No data found when reloading db")
           {:keys [schema initial-data]} (dt/read-transit-str data)
           conn (try

src/main/frontend/db/rtc/debug_ui.cljs

@@ -1,13 +1,14 @@
 (ns frontend.db.rtc.debug-ui
   "Debug UI for rtc module"
   (:require [fipp.edn :as fipp]
-            [frontend.common.missionary-util :as c.m]
             [frontend.db :as db]
+            [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.handler.user :as user]
             [frontend.persist-db.browser :as db-browser]
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [logseq.shui.ui :as shui]
             [missionary.core :as m]
@@ -15,7 +16,6 @@
             [rum.core :as rum]))
 
 (defonce debug-state (:rtc/state @state/state))
-(defonce rtc-log-flow (m/watch (:rtc/log @state/state)))
 
 (defn- stop
   []
@@ -37,7 +37,7 @@
                                          logs)]
                              (reset! (get state ::logs) logs*)
                              logs*))
-                         nil rtc-log-flow)
+                         nil rtc-flows/rtc-log-flow)
                         ::sub-logs)]
                    (reset! (get state ::sub-log-canceler) canceler)
                    state))
@@ -102,6 +102,7 @@
             :remote-graphs (:remote-graphs debug-state*)
             :online-users (:online-users debug-state*)
             :auto-push? (:auto-push? debug-state*)
+            :remote-profile? (:remote-profile? debug-state*)
             :current-page (state/get-current-page)
             :blocks-count (when-let [page (state/get-current-page)]
                             (count (:block/_page (db/get-page page))))}
@@ -126,7 +127,15 @@
                               {:on-click
                                (fn []
                                  (let [^object worker @db-browser/*worker]
-                                   (.rtc-toggle-auto-push worker (state/get-current-repo))))})]
+                                   (.rtc-toggle-auto-push worker)))})]
+        [:div.mr-2 (ui/button (str "Toggle remote profile("
+                                   (if (:remote-profile? debug-state*)
+                                     "ON" "OFF")
+                                   ")")
+                              {:on-click
+                               (fn []
+                                 (let [^object worker @db-browser/*worker]
+                                   (.rtc-toggle-remote-profile worker)))})]
         [:div (shui/button
                {:variant :outline
                 :class "text-red-rx-09 border-red-rx-08 hover:text-red-rx-10"

src/main/frontend/extensions/fsrs.cljs

@@ -1,7 +1,6 @@
 (ns frontend.extensions.fsrs
   "Flashcards functions based on FSRS, only works in db-based graphs"
   (:require [clojure.string :as string]
-            [frontend.common.missionary-util :as c.m]
             [frontend.components.block :as component-block]
             [frontend.config :as config]
             [frontend.context.i18n :refer [t]]
@@ -17,7 +16,9 @@
             [frontend.state :as state]
             [frontend.ui :as ui]
             [frontend.util :as util]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.shui.ui :as shui]
             [missionary.core :as m]
             [open-spaced-repetition.cljc-fsrs.core :as fsrs.core]
@@ -247,7 +248,7 @@
         all-cards (concat
                    [{:db/id :global
                      :block/title "All cards"}]
-                   (db-model/get-class-objects repo (:db/id (db/entity :logseq.class/Cards))))
+                   (db-model/get-class-objects repo (:db/id (entity-plus/entity-memoized (db/get-db) :logseq.class/Cards))))
         *block-ids (::block-ids state)
         block-ids (rum/react *block-ids)
         loading? (rum/react (::loading? state))
@@ -294,15 +295,15 @@
 (def ^:private new-task--update-due-cards-count
   "Return a task that update `:srs/cards-due-count` periodically."
   (m/sp
-   (let [repo (state/get-current-repo)]
-     (if (config/db-based-graph? repo)
-       (m/?
-        (m/reduce
-         (fn [_ _]
-           (p/let [due-cards (<get-due-card-block-ids repo nil)]
-             (state/set-state! :srs/cards-due-count (count due-cards))))
-         (c.m/clock (* 3600 1000))))
-       (srs/update-cards-due-count!)))))
+    (let [repo (state/get-current-repo)]
+      (if (config/db-based-graph? repo)
+        (m/?
+         (m/reduce
+          (fn [_ _]
+            (p/let [due-cards (<get-due-card-block-ids repo nil)]
+              (state/set-state! :srs/cards-due-count (count due-cards))))
+          (c.m/clock (* 3600 1000))))
+        (srs/update-cards-due-count!)))))
 
 (defn update-due-cards-count
   []

src/main/frontend/flows.cljs

@@ -0,0 +1,17 @@
+(ns frontend.flows
+  "This ns contains some event flows."
+  (:require [missionary.core :as m]))
+
+;; Some Input Atoms
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def *current-repo (atom nil))
+
+;; Public Flows
+;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(def current-repo-flow
+  "Like get-current-repo."
+  (m/eduction
+   (dedupe)
+   (m/watch *current-repo)))

src/main/frontend/handler/assets.cljs

@@ -1,7 +1,6 @@
 (ns ^:no-doc frontend.handler.assets
   (:require [cljs-http-missionary.client :as http]
             [clojure.string :as string]
-            [frontend.common.missionary-util :as c.m]
             [frontend.config :as config]
             [frontend.fs :as fs]
             [frontend.fs.nfs :as nfs]
@@ -9,6 +8,7 @@
             [frontend.state :as state]
             [frontend.util :as util]
             [logseq.common.config :as common-config]
+            [frontend.common.missionary :as c.m]
             [logseq.common.path :as path]
             [logseq.common.util :as common-util]
             [medley.core :as medley]

src/main/frontend/handler/db_based/rtc.cljs

@@ -1,13 +1,13 @@
 (ns frontend.handler.db-based.rtc
   "RTC handler"
   (:require [cljs-time.core :as t]
-            [frontend.common.missionary-util :as c.m]
             [frontend.config :as config]
             [frontend.db :as db]
             [frontend.handler.db-based.rtc-flows :as rtc-flows]
             [frontend.handler.notification :as notification]
             [frontend.handler.user :as user-handler]
             [frontend.state :as state]
+            [frontend.common.missionary :as c.m]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
             [logseq.db.sqlite.common-db :as sqlite-common-db]

src/main/frontend/handler/db_based/rtc_flows.cljs

@@ -1,7 +1,7 @@
 (ns frontend.handler.db-based.rtc-flows
   "Flows related to RTC"
-  (:require [frontend.common.missionary-util :as c.m]
-            [frontend.state :as state]
+  (:require [frontend.state :as state]
+            [frontend.common.missionary :as c.m]
             [logseq.common.util :as common-util]
             [missionary.core :as m]))
 

src/main/frontend/handler/editor.cljs

@@ -2,6 +2,7 @@
   (:require [clojure.set :as set]
             [clojure.string :as string]
             [clojure.walk :as w]
+            [datascript.core :as d]
             [dommy.core :as dom]
             [frontend.commands :as commands]
             [frontend.config :as config]
@@ -54,18 +55,18 @@
             [logseq.common.util.block-ref :as block-ref]
             [logseq.common.util.page-ref :as page-ref]
             [logseq.db :as ldb]
-            [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.frontend.property :as db-property]
+            [logseq.db.frontend.schema :as db-schema]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.mldoc :as gp-mldoc]
             [logseq.graph-parser.property :as gp-property]
             [logseq.graph-parser.text :as text]
             [logseq.graph-parser.utf8 :as utf8]
             [logseq.outliner.core :as outliner-core]
-            [promesa.core :as p]
-            [rum.core :as rum]
             [logseq.outliner.property :as outliner-property]
-            [datascript.core :as d]))
+            [promesa.core :as p]
+            [rum.core :as rum]))
 
 ;; FIXME: should support multiple images concurrently uploading
 
@@ -3457,15 +3458,16 @@
 (defn- db-collapsable?
   [block]
   (let [class-properties (:classes-properties (outliner-property/get-block-classes-properties (db/get-db) (:db/id block)))
-        properties (->> (map :a (d/datoms (db/get-db) :eavt (:db/id block)))
-                        (map db/entity)
+        db (db/get-db)
+        properties (->> (map :a (d/datoms db :eavt (:db/id block)))
+                        (map (partial entity-plus/entity-memoized db))
                         (concat class-properties)
                         (remove (fn [e] (db-property/db-attribute-properties (:db/ident e))))
                         (remove outliner-property/property-with-other-position?)
                         (remove (fn [e] (:hide? (:block/schema e))))
                         (remove nil?))]
     (or (seq properties)
-        (ldb/class-instance? (db/entity :logseq.class/Query) block))))
+        (ldb/class-instance? (entity-plus/entity-memoized db :logseq.class/Query) block))))
 
 (defn collapsable?
   ([block-id]

src/main/frontend/handler/export.cljs

@@ -190,16 +190,13 @@
         (.setAttribute anchor "download" filename)
         (.click anchor)))))
 
-(defn export-repo-as-debug-json!
+(defn export-repo-as-debug-transit!
   [repo]
   (p/let [result (export-common-handler/<get-debug-datoms repo)
-          json-str (-> result
-                       bean/->js
-                       js/JSON.stringify)
-          filename (file-name (str repo "-debug-datoms") :json)
-          data-str (str "data:text/json;charset=utf-8,"
-                        (js/encodeURIComponent json-str))]
-    (when-let [anchor (gdom/getElement "download-as-json-debug")]
+          filename (file-name (str repo "-debug-datoms") :transit)
+          data-str (str "data:text/transit;charset=utf-8,"
+                        (js/encodeURIComponent result))]
+    (when-let [anchor (gdom/getElement "download-as-transit-debug")]
       (.setAttribute anchor "href" data-str)
       (.setAttribute anchor "download" filename)
       (.click anchor))))

src/main/frontend/handler/export/common.cljs

@@ -191,8 +191,7 @@
 (defn <get-debug-datoms
   [repo]
   (when-let [^object worker @db-browser/*worker]
-    (p/let [result (.get-debug-datoms worker repo)]
-      (ldb/read-transit-str result))))
+    (.get-debug-datoms worker repo)))
 
 (defn <get-all-page->content
   [repo]

src/main/frontend/handler/import.cljs

@@ -27,7 +27,8 @@
             [frontend.persist-db :as persist-db]
             [promesa.core :as p]
             [frontend.db.async :as db-async]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db :as ldb]))
 
 (defn index-files!
   "Create file structure, then parse into DB (client only)"
@@ -233,7 +234,7 @@
      (p/do!
       (persist-db/<import-db graph buffer)
       (state/add-repo! {:url graph})
-      (repo-handler/restore-and-setup-repo! graph)
+      (repo-handler/restore-and-setup-repo! graph {:import-type "sqlite"})
       (state/set-current-repo! graph)
       (persist-db/<export-db graph {})
       (db/transact! graph (sqlite-util/import-tx :sqlite-db))
@@ -291,3 +292,15 @@
     (async/go
       (async/<! (import-from-tree! clj-data tree-vec-translate-json))
       (finished-ok-handler nil)))) ;; it was designed to accept a list of imported page names but now deprecated
+
+(defn import-from-debug-transit!
+  [bare-graph-name raw finished-ok-handler]
+  (let [graph (str config/db-version-prefix bare-graph-name)
+        datoms (ldb/read-transit-str raw)]
+    (p/do!
+     (persist-db/<new graph {:import-type "debug-transit"
+                             :datoms datoms})
+     (state/add-repo! {:url graph})
+     (repo-handler/restore-and-setup-repo! graph {:import-type "debug-transit"})
+     (state/set-current-repo! graph)
+     (finished-ok-handler nil))))

src/main/frontend/handler/repo.cljs

@@ -66,10 +66,10 @@
 (defn restore-and-setup-repo!
   "Restore the db of a graph from the persisted data, and setup. Create a new
   conn, or replace the conn in state with a new one."
-  [repo]
+  [repo & {:as opts}]
   (p/do!
    (state/set-db-restoring! true)
-   (db-restore/restore-graph! repo)
+   (db-restore/restore-graph! repo opts)
    (repo-config-handler/restore-repo-config! repo)
    (when (config/global-config-enabled?)
      (global-config-handler/restore-global-config!))

src/main/frontend/handler/user.cljs

@@ -6,7 +6,7 @@
             [cljs-time.core :as t]
             [cljs.core.async :as async :refer [<! go]]
             [clojure.string :as string]
-            [frontend.common.missionary-util :as c.m]
+            [frontend.common.missionary :as c.m]
             [frontend.config :as config]
             [frontend.debug :as debug]
             [frontend.handler.config :as config-handler]

src/main/frontend/persist_db/browser.cljs

@@ -190,13 +190,13 @@
     (when-let [^js sqlite @*worker]
       (.releaseAccessHandles sqlite repo)))
 
-  (<fetch-initial-data [_this repo _opts]
+  (<fetch-initial-data [_this repo opts]
     (when-let [^js sqlite @*worker]
       (-> (p/let [db-exists? (.dbExists sqlite repo)
                   disk-db-data (when-not db-exists? (ipc/ipc :db-get repo))
                   _ (when disk-db-data
                       (.importDb sqlite repo disk-db-data))
-                  _ (.createOrOpenDB sqlite repo (ldb/write-transit-str {}))]
+                  _ (.createOrOpenDB sqlite repo (ldb/write-transit-str opts))]
             (.getInitialData sqlite repo))
           (p/catch sqlite-error-handler))))
 

src/main/frontend/state.cljs

@@ -11,6 +11,7 @@
             [electron.ipc :as ipc]
             [frontend.db.conn-state :as db-conn-state]
             [frontend.db.transact :as db-transact]
+            [frontend.flows :as flows]
             [frontend.mobile.util :as mobile-util]
             [frontend.rum :as r]
             [frontend.spec.storage :as storage-spec]
@@ -21,6 +22,7 @@
             [goog.object :as gobj]
             [logseq.common.config :as common-config]
             [logseq.db :as ldb]
+            [logseq.db.frontend.entity-plus :as entity-plus]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.shui.dialog.core :as shui-dialog]
             [logseq.shui.ui :as shui]
@@ -599,7 +601,7 @@ should be done through this fn in order to get global config and config defaults
   (let [repo (get-current-repo)]
     (if (sqlite-util/db-based-graph? repo)
       (when-let [conn (db-conn-state/get-conn repo)]
-        (get (d/entity @conn :logseq.class/Journal)
+        (get (entity-plus/entity-memoized @conn :logseq.class/Journal)
              :logseq.property.journal/title-format
              "MMM do, yyyy"))
       (common-config/get-date-formatter (get-config)))))
@@ -978,6 +980,7 @@ Similar to re-frame subscriptions"
 (defn set-current-repo!
   [repo]
   (swap! state assoc :git/current-repo repo)
+  (reset! flows/*current-repo repo)
   (if repo
     (storage/set :git/current-repo repo)
     (storage/remove :git/current-repo))

src/main/frontend/worker/db/migrate.cljs

@@ -17,6 +17,7 @@
             [clojure.string :as string]
             [logseq.db.frontend.content :as db-content]
             [logseq.common.util.page-ref :as page-ref]
+            [datascript.impl.entity :as de]
             [logseq.common.util.date-time :as date-time-util]
             [cljs-time.coerce :as tc]))
 
@@ -584,6 +585,23 @@
   (when (< db-schema/version max-schema-version)
     (js/console.warn (str "Current db schema-version is " db-schema/version ", max available schema-version is " max-schema-version))))
 
+(defn- ensure-built-in-class-exists!
+  [conn]
+  (let [classes' [:logseq.class/Property :logseq.class/Tag :logseq.class/Page :logseq.class/Journal :logseq.class/Whiteboard]
+        new-classes (->> (select-keys db-class/built-in-classes classes')
+                         ;; class already exists, this should never happen
+                         (remove (fn [[k _]] (d/entity @conn k)))
+                         (into {})
+                         (#(sqlite-create-graph/build-initial-classes* % {}))
+                         (map (fn [b] (assoc b :logseq.property/built-in? true))))
+        new-class-idents (keep (fn [class]
+                                 (when-let [db-ident (:db/ident class)]
+                                   {:db/ident db-ident})) new-classes)
+        tx-data (concat new-class-idents new-classes)]
+    (when (seq tx-data)
+      (d/transact! conn tx-data {:fix-db? true
+                                 :db-migrate? true}))))
+
 (defn- upgrade-version!
   [conn search-db db-based? version {:keys [properties classes fix]}]
   (let [db @conn
@@ -615,6 +633,73 @@
     (ldb/transact! conn tx-data' {:db-migrate? true})
     (println "DB schema migrated to" version)))
 
+(defn fix-path-refs!
+  [conn]
+  (let [data (keep
+              (fn [d]
+                (when (not (de/entity? (d/entity @conn (:v d))))
+                  [:db/retract (:e d) (:a d) (:v d)]))
+              (d/datoms @conn :avet :block/path-refs))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-missing-title!
+  [conn]
+  (let [data (->>
+              (mapcat
+               (fn [d]
+                 (let [entity (d/entity @conn (:e d))]
+                   [(when-not (:block/title entity)
+                      [:db/add (:e d) :block/title (:v d)])
+                    (when-not (:block/uuid entity)
+                      [:db/add (:e d) :block/uuid (d/squuid)])
+                    (when-not (:block/format entity)
+                      [:db/add (:e d) :block/format :markdown])]))
+               (d/datoms @conn :avet :block/name))
+              (remove nil?))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-block-timestamps!
+  [conn]
+  (let [data (map
+              (fn [d]
+                (let [entity (d/entity @conn (:e d))]
+                  (when (or (nil? (:block/created-at entity))
+                            (nil? (:block/updated-at entity)))
+                    (-> (select-keys entity [:db/id :block/created-at :block/updated-at])
+                        sqlite-util/block-with-timestamps))))
+              (d/datoms @conn :avet :block/uuid))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
+(defn fix-properties!
+  [conn]
+  (let [schema (:schema @conn)
+        wrong-properties (filter (fn [[k v]]
+                                   (and (int? k) (not (qualified-ident? v)))) schema)
+        data (map (fn [[k _v]]
+                    [:db/retract k :db/valueType]) wrong-properties)]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true})
+      (d/reset-schema! conn (apply dissoc schema (keys wrong-properties))))))
+
+(defn fix-missing-page-tag!
+  [conn]
+  (let [data (keep
+              (fn [d]
+                (let [entity (d/entity @conn (:e d))]
+                  (when-not (:block/tags entity)
+                    [:db/add (:e d) :block/tags :logseq.class/Page])))
+              (d/datoms @conn :avet :block/name))]
+    (when (seq data)
+      (ldb/transact! conn data {:fix-db? true
+                                :db-migrate? true}))))
+
 (defn migrate
   "Migrate 'frontend' datascript schema and data. To add a new migration,
   add an entry to schema-version->updates and bump db-schema/version"
@@ -637,14 +722,28 @@
                                 (when (and (< version-in-db v) (<= v db-schema/version))
                                   [v updates]))
                               schema-version->updates)]
+            (fix-path-refs! conn)
+            (fix-missing-title! conn)
+            (fix-properties! conn)
+            (fix-block-timestamps! conn)
             (println "DB schema migrated from" version-in-db)
             (doseq [[v m] updates]
-              (upgrade-version! conn search-db db-based? v m)))
+              (upgrade-version! conn search-db db-based? v m))
+            (fix-missing-page-tag! conn))
           (catch :default e
             (prn :error (str "DB migration failed to migrate to " db-schema/version " from " version-in-db ":"))
             (js/console.error e)
             (throw e)))))))
 
+(defn fix-db!
+  [conn]
+  (ensure-built-in-class-exists! conn)
+  (fix-path-refs! conn)
+  (fix-missing-title! conn)
+  (fix-properties! conn)
+  (fix-block-timestamps! conn)
+  (fix-missing-page-tag! conn))
+
 ;; Backend migrations
 ;; ==================
 

src/main/frontend/worker/db_worker.cljs

@@ -108,7 +108,7 @@
 
 (defn- rebuild-db-from-datoms!
   "Persistent-sorted-set has been broken, used addresses can't be found"
-  [datascript-conn sqlite-db]
+  [datascript-conn sqlite-db import-type]
   (let [datoms (get-all-datoms-from-sqlite-db sqlite-db)
         db (d/init-db [] db-schema/schema-for-db-based-graph
                       {:storage (storage/storage @datascript-conn)})
@@ -117,10 +117,12 @@
                              [:db/add (:e d) (:a d) (:v d) (:t d)]) datoms))]
     (prn :debug :rebuild-db-from-datoms :datoms-count (count datoms))
     ;; export db first
-    (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false])
-    (worker-util/post-message :export-current-db [])
+    (when-not import-type
+      (worker-util/post-message :notification ["The SQLite db will be exported to avoid any data-loss." :warning false])
+      (worker-util/post-message :export-current-db []))
     (.exec sqlite-db #js {:sql "delete from kvs"})
-    (d/reset-conn! datascript-conn db)))
+    (d/reset-conn! datascript-conn db)
+    (db-migrate/fix-db! datascript-conn)))
 
 (comment
   (defn- gc-kvs-table!
@@ -287,7 +289,7 @@
   (.exec db "PRAGMA journal_mode=WAL"))
 
 (defn- create-or-open-db!
-  [repo {:keys [config import-type]}]
+  [repo {:keys [config import-type datoms]}]
   (when-not (worker-state/get-sqlite-conn repo)
     (p/let [[db search-db client-ops-db :as dbs] (get-dbs repo)
             storage (new-sqlite-storage repo {})
@@ -305,12 +307,19 @@
       (search/create-tables-and-triggers! search-db)
       (let [schema (sqlite-util/get-schema repo)
             conn (sqlite-common-db/get-storage-conn storage schema)
-            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn client-ops-storage client-op/schema-in-db))
-            initial-data-exists? (and (d/entity @conn :logseq.class/Root)
-                                      (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type))))]
+            _ (when datoms
+                (let [data (map (fn [datom]
+                                  [:db/add (:e datom) (:a datom) (:v datom)]) datoms)]
+                  (d/transact! conn data {:initial-db? true})))
+            client-ops-conn (when-not @*publishing? (sqlite-common-db/get-storage-conn
+                                                     client-ops-storage
+                                                     client-op/schema-in-db))
+            initial-data-exists? (when (nil? datoms)
+                                   (and (d/entity @conn :logseq.class/Root)
+                                        (= "db" (:kv/value (d/entity @conn :logseq.kv/db-type)))))]
         (swap! *datascript-conns assoc repo conn)
         (swap! *client-ops-conns assoc repo client-ops-conn)
-        (when (and db-based? (not initial-data-exists?))
+        (when (and db-based? (not initial-data-exists?) (not datoms))
           (let [config (or config "")
                 initial-data (sqlite-create-graph/build-db-initial-data config
                                                                         (when import-type {:import-type import-type}))]
@@ -327,7 +336,7 @@
           (db-migrate/migrate conn search-db)
           (catch :default _e
             (when db-based?
-              (rebuild-db-from-datoms! conn db))))
+              (rebuild-db-from-datoms! conn db import-type))))
 
         (db-listener/listen-db-changes! repo (get @*datascript-conns repo))))))
 
@@ -729,7 +738,8 @@
   (get-debug-datoms
    [this repo]
    (when-let [db (worker-state/get-sqlite-conn repo)]
-     (ldb/write-transit-str (worker-export/get-debug-datoms db))))
+     (let [conn (worker-state/get-datascript-conn repo)]
+       (ldb/write-transit-str (worker-export/get-debug-datoms conn db)))))
 
   (get-all-pages
    [this repo]
@@ -755,6 +765,10 @@
    [this]
    (rtc-core/rtc-toggle-auto-push))
 
+  (rtc-toggle-remote-profile
+   [this]
+   (rtc-core/rtc-toggle-remote-profile))
+
   (rtc-grant-graph-access
    [this token graph-uuid target-user-uuids-str target-user-emails-str]
    (let [target-user-uuids (ldb/read-transit-str target-user-uuids-str)

src/main/frontend/worker/device.cljs

@@ -4,12 +4,12 @@
             [cljs-time.coerce :as tc]
             [cljs-time.core :as t]
             [clojure.string :as string]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.crypt :as crypt]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.ws-util :as ws-util]
             [frontend.worker.state :as worker-state]
             [goog.crypt.base64 :as base64]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [missionary.core :as m]
             [promesa.core :as p]))
@@ -95,81 +95,81 @@
   Import to `*device-id`, `*device-public-key`, `*device-private-key`"
   [token]
   (m/sp
-   (let [device-uuid (c.m/<? (<get-item item-key-device-id))]
-     (when-not device-uuid
-       (let [get-ws-create-task (new-get-ws-create-task token)
-             agent-data (js->clj (.toJSON js/navigator.userAgentData) :keywordize-keys true)
-             generated-device-name (string/join
-                                    "-"
-                                    [(:platform agent-data)
-                                     (when (:mobile agent-data) "mobile")
-                                     (:brand (first (:brands agent-data)))
-                                     (tc/to-epoch (t/now))])
-             {:keys [device-id device-name created-at updated-at]}
-             (m/? (new-task--add-user-device get-ws-create-task generated-device-name))
-             {:keys [publicKey privateKey]} (c.m/<? (crypt/<gen-key-pair))
-             public-key-jwk (c.m/<? (crypt/<export-key publicKey))
-             private-key-jwk (c.m/<? (crypt/<export-key privateKey))]
-         (c.m/<? (<set-item! item-key-device-id (str device-id)))
-         (c.m/<? (<set-item! item-key-device-name device-name))
-         (c.m/<? (<set-item! item-key-device-created-at created-at))
-         (c.m/<? (<set-item! item-key-device-updated-at updated-at))
-         (c.m/<? (<set-item! item-key-device-public-key-jwk public-key-jwk))
-         (c.m/<? (<set-item! item-key-device-private-key-jwk private-key-jwk))
-         (m/? (new-task--add-device-public-key
-               get-ws-create-task device-id "default-public-key" public-key-jwk))))
-     (c.m/<?
-      (p/let [device-uuid-str (<get-item item-key-device-id)
-              device-name (<get-item item-key-device-name)
-              device-public-key-jwk (<get-item item-key-device-public-key-jwk)
-              device-public-key (crypt/<import-public-key device-public-key-jwk)
-              device-private-key-jwk (<get-item item-key-device-private-key-jwk)
-              device-private-key (crypt/<import-private-key device-private-key-jwk)]
-        (reset! *device-id (uuid device-uuid-str))
-        (reset! *device-name device-name)
-        (reset! *device-public-key device-public-key)
-        (reset! *device-private-key device-private-key))))))
+    (let [device-uuid (c.m/<? (<get-item item-key-device-id))]
+      (when-not device-uuid
+        (let [get-ws-create-task (new-get-ws-create-task token)
+              agent-data (js->clj (.toJSON js/navigator.userAgentData) :keywordize-keys true)
+              generated-device-name (string/join
+                                     "-"
+                                     [(:platform agent-data)
+                                      (when (:mobile agent-data) "mobile")
+                                      (:brand (first (:brands agent-data)))
+                                      (tc/to-epoch (t/now))])
+              {:keys [device-id device-name created-at updated-at]}
+              (m/? (new-task--add-user-device get-ws-create-task generated-device-name))
+              {:keys [publicKey privateKey]} (c.m/<? (crypt/<gen-key-pair))
+              public-key-jwk (c.m/<? (crypt/<export-key publicKey))
+              private-key-jwk (c.m/<? (crypt/<export-key privateKey))]
+          (c.m/<? (<set-item! item-key-device-id (str device-id)))
+          (c.m/<? (<set-item! item-key-device-name device-name))
+          (c.m/<? (<set-item! item-key-device-created-at created-at))
+          (c.m/<? (<set-item! item-key-device-updated-at updated-at))
+          (c.m/<? (<set-item! item-key-device-public-key-jwk public-key-jwk))
+          (c.m/<? (<set-item! item-key-device-private-key-jwk private-key-jwk))
+          (m/? (new-task--add-device-public-key
+                get-ws-create-task device-id "default-public-key" public-key-jwk))))
+      (c.m/<?
+       (p/let [device-uuid-str (<get-item item-key-device-id)
+               device-name (<get-item item-key-device-name)
+               device-public-key-jwk (<get-item item-key-device-public-key-jwk)
+               device-public-key (crypt/<import-public-key device-public-key-jwk)
+               device-private-key-jwk (<get-item item-key-device-private-key-jwk)
+               device-private-key (crypt/<import-private-key device-private-key-jwk)]
+         (reset! *device-id (uuid device-uuid-str))
+         (reset! *device-name device-name)
+         (reset! *device-public-key device-public-key)
+         (reset! *device-private-key device-private-key))))))
 
 (defn new-task--list-devices
   "Return device list.
   Also sync local device metadata to remote if not exists in remote side"
   [token]
   (m/sp
-   (let [get-ws-create-task (new-get-ws-create-task token)
-         devices (m/? (new-task--get-user-devices get-ws-create-task))]
-     (when
+    (let [get-ws-create-task (new-get-ws-create-task token)
+          devices (m/? (new-task--get-user-devices get-ws-create-task))]
+      (when
           ;; check current device has been synced to remote
           ;; if not exists in remote, remove local-metadata and recreate in local and remote
-      (and @*device-id @*device-name @*device-public-key
-           (not (some
-                 (fn [device]
-                   (let [{:keys [device-id]} device]
-                     (when (= device-id (str @*device-id))
-                       true)))
-                 devices)))
-       (c.m/<? (<remove-item! item-key-device-id))
-       (c.m/<? (<remove-item! item-key-device-name))
-       (c.m/<? (<remove-item! item-key-device-created-at))
-       (c.m/<? (<remove-item! item-key-device-updated-at))
-       (c.m/<? (<remove-item! item-key-device-public-key-jwk))
-       (c.m/<? (<remove-item! item-key-device-private-key-jwk))
-       (m/? (new-task--ensure-device-metadata! token)))
-     devices)))
+       (and @*device-id @*device-name @*device-public-key
+            (not (some
+                  (fn [device]
+                    (let [{:keys [device-id]} device]
+                      (when (= device-id (str @*device-id))
+                        true)))
+                  devices)))
+        (c.m/<? (<remove-item! item-key-device-id))
+        (c.m/<? (<remove-item! item-key-device-name))
+        (c.m/<? (<remove-item! item-key-device-created-at))
+        (c.m/<? (<remove-item! item-key-device-updated-at))
+        (c.m/<? (<remove-item! item-key-device-public-key-jwk))
+        (c.m/<? (<remove-item! item-key-device-private-key-jwk))
+        (m/? (new-task--ensure-device-metadata! token)))
+      devices)))
 
 (defn new-task--remove-device-public-key
   [token device-uuid key-name]
   (assert (some? key-name))
   (m/sp
-   (when-let [device-uuid* (cond-> device-uuid (string? device-uuid) parse-uuid)]
-     (let [get-ws-create-task (new-get-ws-create-task token)]
-       (m/? (new-task--remove-device-public-key* get-ws-create-task device-uuid* key-name))))))
+    (when-let [device-uuid* (cond-> device-uuid (string? device-uuid) parse-uuid)]
+      (let [get-ws-create-task (new-get-ws-create-task token)]
+        (m/? (new-task--remove-device-public-key* get-ws-create-task device-uuid* key-name))))))
 
 (defn new-task--remove-device
   [token device-uuid]
   (m/sp
-   (when-let [device-uuid* (cond-> device-uuid (string? device-uuid) parse-uuid)]
-     (let [get-ws-create-task (new-get-ws-create-task token)]
-       (m/? (new-task--remove-user-device* get-ws-create-task device-uuid*))))))
+    (when-let [device-uuid* (cond-> device-uuid (string? device-uuid) parse-uuid)]
+      (let [get-ws-create-task (new-get-ws-create-task token)]
+        (m/? (new-task--remove-user-device* get-ws-create-task device-uuid*))))))
 
 (defn new-task--sync-current-graph-encrypted-aes-key
   [token device-uuids-transit-str]
@@ -177,32 +177,32 @@
         device-uuids (ldb/read-transit-str device-uuids-transit-str)]
     (assert (and (seq device-uuids) (every? uuid? device-uuids)) device-uuids)
     (m/sp
-     (when-let [graph-uuid (client-op/get-graph-uuid repo)]
-       (when-let [{:keys [aes-key-jwk]} (crypt/get-graph-keys-jwk repo)]
-         (let [device-uuids (set device-uuids)
-               get-ws-create-task (new-get-ws-create-task token)
-               devices (m/? (new-task--get-user-devices get-ws-create-task))]
-           (when-let [devices* (not-empty
-                                (filter
-                                 (fn [device]
-                                   (and (contains? device-uuids (uuid (:device-id device)))
-                                        (some? (get-in device [:keys :default-public-key]))))
-                                 devices))]
-             (let [device-uuid->encrypted-aes-key
-                   (m/?
-                    (apply m/join (fn [& x] (into {} x))
-                           (map (fn [device]
-                                  (m/sp
-                                   (let [device-public-key
-                                         (c.m/<?
-                                          (crypt/<import-public-key
-                                           (clj->js
-                                            (ldb/read-transit-str
-                                             (get-in device [:keys :default-public-key :public-key])))))]
-                                     [(uuid (:device-id device))
-                                      (base64/encodeByteArray
-                                       (js/Uint8Array.
-                                        (c.m/<? (crypt/<rsa-encrypt aes-key-jwk device-public-key))))])))
-                                devices*)))]
-               (m/? (new-task--sync-encrypted-aes-key*
-                     get-ws-create-task device-uuid->encrypted-aes-key graph-uuid))))))))))
+      (when-let [graph-uuid (client-op/get-graph-uuid repo)]
+        (when-let [{:keys [aes-key-jwk]} (crypt/get-graph-keys-jwk repo)]
+          (let [device-uuids (set device-uuids)
+                get-ws-create-task (new-get-ws-create-task token)
+                devices (m/? (new-task--get-user-devices get-ws-create-task))]
+            (when-let [devices* (not-empty
+                                 (filter
+                                  (fn [device]
+                                    (and (contains? device-uuids (uuid (:device-id device)))
+                                         (some? (get-in device [:keys :default-public-key]))))
+                                  devices))]
+              (let [device-uuid->encrypted-aes-key
+                    (m/?
+                     (apply m/join (fn [& x] (into {} x))
+                            (map (fn [device]
+                                   (m/sp
+                                     (let [device-public-key
+                                           (c.m/<?
+                                            (crypt/<import-public-key
+                                             (clj->js
+                                              (ldb/read-transit-str
+                                               (get-in device [:keys :default-public-key :public-key])))))]
+                                       [(uuid (:device-id device))
+                                        (base64/encodeByteArray
+                                         (js/Uint8Array.
+                                          (c.m/<? (crypt/<rsa-encrypt aes-key-jwk device-public-key))))])))
+                                 devices*)))]
+                (m/? (new-task--sync-encrypted-aes-key*
+                      get-ws-create-task device-uuid->encrypted-aes-key graph-uuid))))))))))

src/main/frontend/worker/export.cljs

@@ -53,7 +53,7 @@
                  (common-file/block->content repo db (:block/uuid e) {} {})])))))
 
 (defn get-debug-datoms
-  [^Object db]
+  [conn ^Object db]
   (some->> (.exec db #js {:sql "select content from kvs"
                           :rowMode "array"})
            bean/->clj
@@ -61,26 +61,10 @@
                      (let [result (sqlite-util/transit-read (first result))]
                        (when (map? result)
                          (:keys result)))))
-           (group-by first)
-           (mapcat (fn [[_id col]]
-                     (let [ident (some (fn [[_e a v _t]]
-                                         (when (= a :db/ident)
-                                           v)) col)
-                           journal (some (fn [[_e a v _t]]
-                                           (when (= a :block/journal-day)
-                                             v)) col)]
-                       (map
-                        (fn [[e a v t]]
-                          (cond
-                            (and (contains? #{:block/title :block/name} a)
-                                 (not (or ident journal)))
-                            [e a (str "debug " e) t]
-
-                            (= a :block/uuid)
-                            [e a (str v) t]
-
-                            :else
-                            [e a v t]))
-                        col))))
-           (distinct)
-           (sort-by first)))
+           (map (fn [[e a v t]]
+                  (if (and (contains? #{:block/title :block/name} a)
+                           (let [entity (d/entity @conn e)]
+                             (and (not (:db/ident entity))
+                                  (not (ldb/journal? entity)))))
+                    (d/datom e a (str "debug " e) t)
+                    (d/datom e a v t))))))

src/main/frontend/worker/handler/page/db_based/page.cljs

@@ -7,15 +7,16 @@
             [logseq.common.util.namespace :as ns-util]
             [logseq.db :as ldb]
             [logseq.db.frontend.class :as db-class]
+            [logseq.db.frontend.entity-plus :as entity-plus]
+            [logseq.db.frontend.entity-util :as entity-util]
+            [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.order :as db-order]
             [logseq.db.frontend.property.build :as db-property-build]
             [logseq.db.frontend.property.util :as db-property-util]
             [logseq.db.sqlite.util :as sqlite-util]
             [logseq.graph-parser.block :as gp-block]
             [logseq.graph-parser.text :as text]
-            [logseq.outliner.validate :as outliner-validate]
-            [logseq.db.frontend.entity-util :as entity-util]
-            [logseq.db.frontend.malli-schema :as db-malli-schema]))
+            [logseq.outliner.validate :as outliner-validate]))
 
 (defn- build-page-tx [db properties page {:keys [whiteboard? class? tags]}]
   (when (:block/uuid page)
@@ -168,7 +169,7 @@
            persist-op?              true
            skip-existing-page-check? false}
     :as options}]
-  (let [date-formatter (:logseq.property.journal/title-format (d/entity db :logseq.class/Journal))
+  (let [date-formatter (:logseq.property.journal/title-format (entity-plus/entity-memoized db :logseq.class/Journal))
         title (sanitize-title title*)
         types (cond class?
                     #{:logseq.class/Tag}

src/main/frontend/worker/rtc/asset.cljs

@@ -7,11 +7,11 @@
     indicates need to upload the asset to server"
   (:require [clojure.set :as set]
             [datascript.core :as d]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
             [frontend.worker.rtc.ws-util :as ws-util]
             [frontend.worker.state :as worker-state]
+            [frontend.common.missionary :as c.m]
             [logseq.common.path :as path]
             [logseq.db :as ldb]
             [malli.core :as ma]

src/main/frontend/worker/rtc/client.cljs

@@ -2,7 +2,6 @@
   "Fns about push local updates"
   (:require [clojure.string :as string]
             [datascript.core :as d]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.rtc.exception :as r.ex]
@@ -11,6 +10,7 @@
             [frontend.worker.rtc.skeleton :as r.skeleton]
             [frontend.worker.rtc.ws :as ws]
             [frontend.worker.rtc.ws-util :as ws-util]
+            [frontend.common.missionary :as c.m]
             [missionary.core :as m]))
 
 (defn- register-graph-updates
@@ -326,7 +326,7 @@
 
 (defn new-task--push-local-ops
   "Return a task: push local updates"
-  [repo conn graph-uuid date-formatter get-ws-create-task add-log-fn]
+  [repo conn graph-uuid date-formatter get-ws-create-task *remote-profile? add-log-fn]
   (m/sp
     (let [block-ops-map-coll (client-op/get&remove-all-block-ops repo)]
       (when-let [block-uuid->remote-ops (not-empty (gen-block-uuid->remote-ops @conn block-ops-map-coll))]
@@ -336,8 +336,9 @@
           (let [local-tx (client-op/get-local-tx repo)
                 r (try
                     (m/? (ws-util/send&recv get-ws-create-task
-                                            {:action "apply-ops" :graph-uuid graph-uuid
-                                             :ops ops-for-remote :t-before (or local-tx 1)}))
+                                            (cond-> {:action "apply-ops" :graph-uuid graph-uuid
+                                                     :ops ops-for-remote :t-before (or local-tx 1)}
+                                              (true? @*remote-profile?) (assoc :profile true))))
                     (catch :default e
                       (rollback repo block-ops-map-coll)
                       (throw e)))]

src/main/frontend/worker/rtc/client_op.cljs

@@ -1,9 +1,9 @@
 (ns frontend.worker.rtc.client-op
   "Store client-ops in a persisted datascript"
   (:require [datascript.core :as d]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.state :as worker-state]
+            [frontend.common.missionary :as c.m]
             [logseq.db.sqlite.util :as sqlite-util]
             [malli.core :as ma]
             [malli.transform :as mt]

src/main/frontend/worker/rtc/const.cljs

@@ -78,6 +78,7 @@
   "TODO: split this mix schema to multiple ones"
   [:map
    [:req-id :string]
+   [:profile {:optional true} :map]
    [:t {:optional true} :int]
    [:t-before {:optional true} :int]
    [:failed-ops {:optional true} [:sequential to-ws-op-schema]]
@@ -172,7 +173,9 @@
        (fn [api-schema]
          (let [[api-name [type']] api-schema]
            (if (= :map type')
-             [api-name (vec (concat (second api-schema) [[:req-id :string] [:action :string]]))]
+             [api-name (vec (concat (second api-schema) [[:req-id :string]
+                                                         [:action :string]
+                                                         [:profile {:optional true} :boolean]]))]
              api-schema)))
        api-schema-seq)))))
 
@@ -190,12 +193,14 @@
         [:map
          [:req-id :string]
          [:action :string]
+         [:profile {:optional true} :boolean]
          [:graph-uuid :string]
          [:ops [:sequential to-ws-op-schema]]
          [:t-before :int]]
         [:map
          [:req-id :string]
          [:action :string]
+         [:profile {:optional true} :boolean]
          [:s3-key :string]]]]
       ["presign-put-temp-s3-obj"
        [:map]]

src/main/frontend/worker/rtc/core.cljs

@@ -2,7 +2,6 @@
   "Main(use missionary) ns for rtc related fns"
   (:require [clojure.data :as data]
             [datascript.core :as d]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.device :as worker-device]
             [frontend.worker.rtc.asset :as r.asset]
             [frontend.worker.rtc.client :as r.client]
@@ -17,6 +16,7 @@
             [frontend.worker.state :as worker-state]
             [frontend.worker.util :as worker-util]
             [logseq.common.config :as common-config]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [malli.core :as ma]
             [missionary.core :as m])
@@ -180,6 +180,7 @@
    & {:keys [auto-push? debug-ws-url] :or {auto-push? true}}]
   (let [ws-url                     (or debug-ws-url (ws-util/get-ws-url token))
         *auto-push?                (atom auto-push?)
+        *remote-profile?           (atom false)
         *last-calibrate-t          (atom nil)
         *online-users              (atom nil)
         *assets-sync-loop-canceler (atom nil)
@@ -195,10 +196,11 @@
         (r.asset/create-assets-sync-loop repo get-ws-create-task graph-uuid conn *auto-push?)
         mixed-flow                 (create-mixed-flow repo get-ws-create-task *auto-push? *online-users)]
     (assert (some? *current-ws))
-    {:rtc-state-flow     (create-rtc-state-flow (create-ws-state-flow *current-ws))
-     :*rtc-auto-push?    *auto-push?
-     :*online-users      *online-users
-     :onstarted-task     started-dfv
+    {:rtc-state-flow       (create-rtc-state-flow (create-ws-state-flow *current-ws))
+     :*rtc-auto-push?      *auto-push?
+     :*rtc-remote-profile? *remote-profile?
+     :*online-users        *online-users
+     :onstarted-task       started-dfv
      :rtc-loop-task
      (holding-rtc-lock
       started-dfv
@@ -225,7 +227,7 @@
                :local-update-check
                (m/? (r.client/new-task--push-local-ops
                      repo conn graph-uuid date-formatter
-                     get-ws-create-task add-log-fn))
+                     get-ws-create-task *remote-profile? add-log-fn))
 
                :online-users-updated
                (reset! *online-users (:online-users (:value event)))
@@ -246,16 +248,21 @@
             (when @*assets-sync-loop-canceler (@*assets-sync-loop-canceler))))))}))
 
 (def ^:private empty-rtc-loop-metadata
-  {:graph-uuid nil
+  {:repo nil
+   :graph-uuid nil
    :user-uuid nil
    :rtc-state-flow nil
    :*rtc-auto-push? nil
+   :*rtc-remote-profile? nil
    :*online-users nil
    :*rtc-lock nil
    :canceler nil
    :*last-stop-exception nil})
 
-(defonce ^:private *rtc-loop-metadata (atom empty-rtc-loop-metadata))
+(defonce ^:private *rtc-loop-metadata (atom empty-rtc-loop-metadata
+                                            :validator
+                                            (fn [v] (= (set (keys empty-rtc-loop-metadata))
+                                                       (set (keys v))))))
 
 ;;; ================ API ================
 (defn new-task--rtc-start
@@ -268,7 +275,7 @@
         (let [user-uuid (:sub (worker-util/parse-jwt token))
               config (worker-state/get-config repo)
               date-formatter (common-config/get-date-formatter config)
-              {:keys [rtc-state-flow *rtc-auto-push? rtc-loop-task *online-users onstarted-task]}
+              {:keys [rtc-state-flow *rtc-auto-push? *rtc-remote-profile? rtc-loop-task *online-users onstarted-task]}
               (create-rtc-loop graph-uuid repo conn date-formatter token)
               *last-stop-exception (atom nil)
               canceler (c.m/run-task rtc-loop-task :rtc-loop-task
@@ -283,6 +290,7 @@
                                             :user-uuid user-uuid
                                             :rtc-state-flow rtc-state-flow
                                             :*rtc-auto-push? *rtc-auto-push?
+                                            :*rtc-remote-profile? *rtc-remote-profile?
                                             :*online-users *online-users
                                             :*rtc-lock *rtc-lock
                                             :canceler canceler
@@ -303,6 +311,11 @@
   (when-let [*auto-push? (:*rtc-auto-push? @*rtc-loop-metadata)]
     (swap! *auto-push? not)))
 
+(defn rtc-toggle-remote-profile
+  []
+  (when-let [*rtc-remote-profile? (:*rtc-remote-profile? @*rtc-loop-metadata)]
+    (swap! *rtc-remote-profile? not)))
+
 (defn new-task--get-graphs
   [token]
   (let [{:keys [get-ws-create-task]} (gen-get-ws-create-map--memoized (ws-util/get-ws-url token))]
@@ -356,14 +369,15 @@
   (let [rtc-loop-metadata-flow (m/watch *rtc-loop-metadata)]
     (m/ap
       (let [{rtc-lock :*rtc-lock
-             :keys [repo graph-uuid user-uuid rtc-state-flow *rtc-auto-push? *online-users
-                    *last-stop-exception]}
+             :keys [repo graph-uuid user-uuid rtc-state-flow *rtc-auto-push? *rtc-remote-profile?
+                    *online-users *last-stop-exception]}
             (m/?< rtc-loop-metadata-flow)]
         (try
           (when (and repo rtc-state-flow *rtc-auto-push? rtc-lock)
             (m/?<
              (m/latest
-              (fn [rtc-state rtc-auto-push? rtc-lock online-users pending-local-ops-count local-tx remote-tx]
+              (fn [rtc-state rtc-auto-push? rtc-remote-profile?
+                   rtc-lock online-users pending-local-ops-count local-tx remote-tx]
                 {:graph-uuid graph-uuid
                  :user-uuid user-uuid
                  :unpushed-block-update-count pending-local-ops-count
@@ -372,9 +386,12 @@
                  :rtc-state rtc-state
                  :rtc-lock rtc-lock
                  :auto-push? rtc-auto-push?
+                 :remote-profile? rtc-remote-profile?
                  :online-users online-users
                  :last-stop-exception-ex-data (some-> *last-stop-exception deref ex-data)})
-              rtc-state-flow (m/watch *rtc-auto-push?) (m/watch rtc-lock) (m/watch *online-users)
+              rtc-state-flow
+              (m/watch *rtc-auto-push?) (m/watch *rtc-remote-profile?)
+              (m/watch rtc-lock) (m/watch *online-users)
               (client-op/create-pending-block-ops-count-flow repo)
               (rtc-log-and-state/create-local-t-flow graph-uuid)
               (rtc-log-and-state/create-remote-t-flow graph-uuid))))

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -4,7 +4,6 @@
   (:require [cljs-http-missionary.client :as http]
             [clojure.set :as set]
             [datascript.core :as d]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.crypt :as crypt]
             [frontend.worker.db-listener :as db-listener]
             [frontend.worker.rtc.client-op :as client-op]
@@ -13,6 +12,7 @@
             [frontend.worker.rtc.ws-util :as ws-util]
             [frontend.worker.state :as worker-state]
             [frontend.worker.util :as worker-util]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [logseq.db.frontend.malli-schema :as db-malli-schema]
             [logseq.db.frontend.schema :as db-schema]

src/main/frontend/worker/rtc/log_and_state.cljs

@@ -1,8 +1,8 @@
 (ns frontend.worker.rtc.log-and-state
   "Fns to generate rtc related logs"
-  (:require [frontend.common.missionary-util :as c.m]
-            [frontend.common.schema-register :as sr]
+  (:require [frontend.common.schema-register :as sr]
             [frontend.worker.util :as worker-util]
+            [frontend.common.missionary :as c.m]
             [malli.core :as ma]
             [missionary.core :as m]))
 

src/main/frontend/worker/rtc/ws.cljs

@@ -3,9 +3,9 @@
   based on
   https://github.com/ReilySiegel/missionary-websocket/blob/master/src/com/reilysiegel/missionary/websocket.cljs"
   (:require [cljs-http-missionary.client :as http]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.rtc.exception :as r.ex]
+            [frontend.common.missionary :as c.m]
             [missionary.core :as m]))
 
 (defn- get-state

src/rtc_e2e_test/client_steps.cljs

@@ -2,32 +2,32 @@
   (:require [cljs.test :as t :refer [is]]
             [const]
             [datascript.core :as d]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.core :as rtc-core]
             [helper]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [missionary.core :as m]))
 
 (def ^:private step0
   {:client1
    (m/sp
-    (let [conn (helper/get-downloaded-test-conn)
-          tx-data (const/tx-data-map :create-page)]
-      (helper/transact! conn tx-data)
-      (is (=
-           #{[:update-page const/page1-uuid]
-             [:update const/page1-uuid
-              [[:block/title "[\"~#'\",\"basic-edits-test\"]" true]
-               [:block/created-at "[\"~#'\",1724836490809]" true]
-               [:block/updated-at "[\"~#'\",1724836490809]" true]
-               [:block/type "[\"~#'\",\"page\"]" true]]]
-             [:move const/block1-uuid]
-             [:update const/block1-uuid
-              [[:block/updated-at "[\"~#'\",1724836490810]" true]
-               [:block/created-at "[\"~#'\",1724836490810]" true]
-               [:block/title "[\"~#'\",\"block1\"]" true]]]}
-           (set (map helper/simplify-client-op (client-op/get-all-block-ops const/downloaded-test-repo)))))))
+     (let [conn (helper/get-downloaded-test-conn)
+           tx-data (const/tx-data-map :create-page)]
+       (helper/transact! conn tx-data)
+       (is (=
+            #{[:update-page const/page1-uuid]
+              [:update const/page1-uuid
+               [[:block/title "[\"~#'\",\"basic-edits-test\"]" true]
+                [:block/created-at "[\"~#'\",1724836490809]" true]
+                [:block/updated-at "[\"~#'\",1724836490809]" true]
+                [:block/type "[\"~#'\",\"page\"]" true]]]
+              [:move const/block1-uuid]
+              [:update const/block1-uuid
+               [[:block/updated-at "[\"~#'\",1724836490810]" true]
+                [:block/created-at "[\"~#'\",1724836490810]" true]
+                [:block/title "[\"~#'\",\"block1\"]" true]]]}
+            (set (map helper/simplify-client-op (client-op/get-all-block-ops const/downloaded-test-repo)))))))
    :client2 nil})
 
 (def ^:private step1
@@ -35,53 +35,53 @@
   client2: start rtc, wait page1, remote->client2"
   {:client1
    (m/sp
-    (let [r (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))]
-      (is (nil? r))
-      (m/? (helper/new-task--wait-all-client-ops-sent))))
+     (let [r (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))]
+       (is (nil? r))
+       (m/? (helper/new-task--wait-all-client-ops-sent))))
    :client2
    (m/sp
-    (let [r (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))]
-      (is (nil? r)))
-    (m/?
-     (c.m/backoff
-      (take 4 c.m/delays)
-      (m/sp
-       (let [conn (helper/get-downloaded-test-conn)
-             page1 (d/pull @conn '[*] [:block/uuid const/page1-uuid])
-             block1 (d/pull @conn '[*] [:block/uuid const/block1-uuid])]
-         (when-not (:block/uuid page1)
-           (throw (ex-info "wait page1 synced" {:missionary/retry true})))
-         (is
-          (= {:block/title "basic-edits-test"
-              :block/name "basic-edits-test"
-              :block/type "page"}
-             (select-keys page1 [:block/title :block/name :block/type])))
-         (is
-          (= {:block/title "block1"
-              :block/order "a0"
-              :block/parent {:db/id (:db/id page1)}}
-             (select-keys block1 [:block/title :block/order :block/parent]))))))))})
+     (let [r (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))]
+       (is (nil? r)))
+     (m/?
+      (c.m/backoff
+       (take 4 c.m/delays)
+       (m/sp
+         (let [conn (helper/get-downloaded-test-conn)
+               page1 (d/pull @conn '[*] [:block/uuid const/page1-uuid])
+               block1 (d/pull @conn '[*] [:block/uuid const/block1-uuid])]
+           (when-not (:block/uuid page1)
+             (throw (ex-info "wait page1 synced" {:missionary/retry true})))
+           (is
+            (= {:block/title "basic-edits-test"
+                :block/name "basic-edits-test"
+                :block/type "page"}
+               (select-keys page1 [:block/title :block/name :block/type])))
+           (is
+            (= {:block/title "block1"
+                :block/order "a0"
+                :block/parent {:db/id (:db/id page1)}}
+               (select-keys block1 [:block/title :block/order :block/parent]))))))))})
 
 (def ^:private step2
   "client1: insert 500 blocks, wait for changes to sync to remote
   client2: wait for blocks to sync from remote"
   {:client1
    (m/sp
-    (let [conn (helper/get-downloaded-test-conn)]
-      (helper/transact! conn (const/tx-data-map :insert-500-blocks))
-      (m/? (helper/new-task--wait-all-client-ops-sent))))
+     (let [conn (helper/get-downloaded-test-conn)]
+       (helper/transact! conn (const/tx-data-map :insert-500-blocks))
+       (m/? (helper/new-task--wait-all-client-ops-sent))))
    :client2
    (c.m/backoff
     (take 4 c.m/delays)
     (m/sp
-     (let [conn (helper/get-downloaded-test-conn)
-           page (d/pull @conn '[*] [:block/uuid const/page2-uuid])]
-       (when-not (:block/uuid page)
-         (throw (ex-info "wait page to be synced" {:missionary/retry true})))
-       (let [blocks (ldb/sort-by-order (ldb/get-page-blocks @conn (:db/id page)))]
-         (is (= 500 (count blocks)))
-         (is (= (map #(str "x" %) (range 500))
-                (map :block/title blocks)))))))})
+      (let [conn (helper/get-downloaded-test-conn)
+            page (d/pull @conn '[*] [:block/uuid const/page2-uuid])]
+        (when-not (:block/uuid page)
+          (throw (ex-info "wait page to be synced" {:missionary/retry true})))
+        (let [blocks (ldb/sort-by-order (ldb/get-page-blocks @conn (:db/id page)))]
+          (is (= 500 (count blocks)))
+          (is (= (map #(str "x" %) (range 500))
+                 (map :block/title blocks)))))))})
 
 (def ^:private step3
   "client1:
@@ -95,32 +95,32 @@
   1. wait the block&its properties to be synced"
   {:client1
    (m/sp
-    (let [conn (helper/get-downloaded-test-conn)
-          tx-data1 (const/tx-data-map :step3-add-task-properties-to-block1)
-          tx-data2 (const/tx-data-map :step3-toggle-status-TODO)
-          tx-data3 (const/tx-data-map :step3-toggle-status-DOING)]
-      (helper/transact! conn tx-data1)
-      (m/? (helper/new-task--wait-all-client-ops-sent))
-      (helper/transact! conn tx-data2)
-      (m/? (helper/new-task--wait-all-client-ops-sent))
-      (helper/transact! conn tx-data3)
-      (m/? (helper/new-task--wait-all-client-ops-sent))))
+     (let [conn (helper/get-downloaded-test-conn)
+           tx-data1 (const/tx-data-map :step3-add-task-properties-to-block1)
+           tx-data2 (const/tx-data-map :step3-toggle-status-TODO)
+           tx-data3 (const/tx-data-map :step3-toggle-status-DOING)]
+       (helper/transact! conn tx-data1)
+       (m/? (helper/new-task--wait-all-client-ops-sent))
+       (helper/transact! conn tx-data2)
+       (m/? (helper/new-task--wait-all-client-ops-sent))
+       (helper/transact! conn tx-data3)
+       (m/? (helper/new-task--wait-all-client-ops-sent))))
    :client2
    (c.m/backoff
     (take 4 c.m/delays)
     (m/sp
-     (let [conn (helper/get-downloaded-test-conn)
-           block1 (d/pull @conn
-                          [{:block/tags [:db/ident]}
-                           {:logseq.task/status [:db/ident]}
-                           {:logseq.task/deadline [:block/journal-day]}]
-                          [:block/uuid const/block1-uuid])]
-       (when-not (= :logseq.task/status.doing (:db/ident (:logseq.task/status block1)))
-         (throw (ex-info "wait block1's task properties to be synced" {:missionary/retry true})))
-       (is (= {:block/tags [{:db/ident :logseq.class/Task}],
-               :logseq.task/status {:db/ident :logseq.task/status.doing}
-               :logseq.task/deadline {:block/journal-day 20240907}}
-              block1)))))})
+      (let [conn (helper/get-downloaded-test-conn)
+            block1 (d/pull @conn
+                           [{:block/tags [:db/ident]}
+                            {:logseq.task/status [:db/ident]}
+                            {:logseq.task/deadline [:block/journal-day]}]
+                           [:block/uuid const/block1-uuid])]
+        (when-not (= :logseq.task/status.doing (:db/ident (:logseq.task/status block1)))
+          (throw (ex-info "wait block1's task properties to be synced" {:missionary/retry true})))
+        (is (= {:block/tags [{:db/ident :logseq.class/Task}],
+                :logseq.task/status {:db/ident :logseq.task/status.doing}
+                :logseq.task/deadline {:block/journal-day 20240907}}
+               block1)))))})
 (def ^:private step4
   "client1:
 
@@ -154,49 +154,49 @@
   - send a message to client1 contains client2's block tree to client1"
   {:client1
    (m/sp
-    (let [conn (helper/get-downloaded-test-conn)
-          tx-data1 (const/tx-data-map :move-blocks-concurrently-1)
-          tx-data2 (const/tx-data-map :move-blocks-concurrently-client1)]
-      (helper/transact! conn tx-data1)
-      (m/? (helper/new-task--wait-all-client-ops-sent))
-      (m/? (helper/new-task--client1-sync-barrier-2->1 "move-blocks-concurrently-signal"))
-      (m/? helper/new-task--stop-rtc)
-      (helper/transact! conn tx-data2)
-      (is (nil? (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))))
-      (m/? (helper/new-task--wait-all-client-ops-sent))
-      (m/? (helper/new-task--client1-sync-barrier-2->1 "step5"))
-      (let [message (m/? (helper/new-task--wait-message-from-other-client
-                          (fn [message] (= "move-blocks-concurrently-page-blocks" (:id message)))
-                          :retry-message "move-blocks-concurrently-page-blocks"))
-            client2-page-blocks (:page-blocks message)
-            client1-page-blocks (ldb/get-page-blocks @conn (:db/id (d/entity @conn [:block/uuid const/page3-uuid]))
-                                                     :pull-keys '[:block/uuid :block/title :block/order
-                                                                  {:block/parent [:block/uuid]}])]
-        (is (= (set client1-page-blocks) (set client2-page-blocks))))))
+     (let [conn (helper/get-downloaded-test-conn)
+           tx-data1 (const/tx-data-map :move-blocks-concurrently-1)
+           tx-data2 (const/tx-data-map :move-blocks-concurrently-client1)]
+       (helper/transact! conn tx-data1)
+       (m/? (helper/new-task--wait-all-client-ops-sent))
+       (m/? (helper/new-task--client1-sync-barrier-2->1 "move-blocks-concurrently-signal"))
+       (m/? helper/new-task--stop-rtc)
+       (helper/transact! conn tx-data2)
+       (is (nil? (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))))
+       (m/? (helper/new-task--wait-all-client-ops-sent))
+       (m/? (helper/new-task--client1-sync-barrier-2->1 "step5"))
+       (let [message (m/? (helper/new-task--wait-message-from-other-client
+                           (fn [message] (= "move-blocks-concurrently-page-blocks" (:id message)))
+                           :retry-message "move-blocks-concurrently-page-blocks"))
+             client2-page-blocks (:page-blocks message)
+             client1-page-blocks (ldb/get-page-blocks @conn (:db/id (d/entity @conn [:block/uuid const/page3-uuid]))
+                                                      :pull-keys '[:block/uuid :block/title :block/order
+                                                                   {:block/parent [:block/uuid]}])]
+         (is (= (set client1-page-blocks) (set client2-page-blocks))))))
    :client2
    (m/sp
-    (let [conn (helper/get-downloaded-test-conn)]
-      (m/?
-       (c.m/backoff
-        (take 4 c.m/delays)
-        (m/sp
-         (let [page3 (d/pull @conn '[*] [:block/uuid const/page3-uuid])
-               page3-blocks (some->> (:db/id page3)
-                                     (ldb/get-page-blocks @conn))]
-           (when-not (:block/uuid page3)
-             (throw (ex-info "wait page3 synced" {:missionary/retry true})))
-           (is (= 6 (count page3-blocks)))))))
-      (m/? (helper/new-task--client2-sync-barrier-2->1 "move-blocks-concurrently-signal"))
-      (m/? helper/new-task--stop-rtc)
-      (helper/transact! conn (const/tx-data-map :move-blocks-concurrently-client2))
-      (is (nil? (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))))
-      (m/? (helper/new-task--wait-all-client-ops-sent))
-      (m/? (helper/new-task--client2-sync-barrier-2->1 "step5"))
-      (m/? (helper/new-task--send-message-to-other-client
-            {:id "move-blocks-concurrently-page-blocks"
-             :page-blocks (ldb/get-page-blocks @conn (:db/id (d/entity @conn [:block/uuid const/page3-uuid]))
-                                               :pull-keys '[:block/uuid :block/title :block/order
-                                                            {:block/parent [:block/uuid]}])}))))})
+     (let [conn (helper/get-downloaded-test-conn)]
+       (m/?
+        (c.m/backoff
+         (take 4 c.m/delays)
+         (m/sp
+           (let [page3 (d/pull @conn '[*] [:block/uuid const/page3-uuid])
+                 page3-blocks (some->> (:db/id page3)
+                                       (ldb/get-page-blocks @conn))]
+             (when-not (:block/uuid page3)
+               (throw (ex-info "wait page3 synced" {:missionary/retry true})))
+             (is (= 6 (count page3-blocks)))))))
+       (m/? (helper/new-task--client2-sync-barrier-2->1 "move-blocks-concurrently-signal"))
+       (m/? helper/new-task--stop-rtc)
+       (helper/transact! conn (const/tx-data-map :move-blocks-concurrently-client2))
+       (is (nil? (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))))
+       (m/? (helper/new-task--wait-all-client-ops-sent))
+       (m/? (helper/new-task--client2-sync-barrier-2->1 "step5"))
+       (m/? (helper/new-task--send-message-to-other-client
+             {:id "move-blocks-concurrently-page-blocks"
+              :page-blocks (ldb/get-page-blocks @conn (:db/id (d/entity @conn [:block/uuid const/page3-uuid]))
+                                                :pull-keys '[:block/uuid :block/title :block/order
+                                                             {:block/parent [:block/uuid]}])}))))})
 
 (def ^:private step6
   "Delete blocks test-1
@@ -214,45 +214,45 @@ client2:
 - check block-tree"
   {:client1
    (m/sp
-    (let [conn (helper/get-downloaded-test-conn)
-          tx-data1 (const/tx-data-map :step6-delete-blocks-client1-1)
-          tx-data2 (const/tx-data-map :step6-delete-blocks-client1-2)]
-      (helper/transact! conn tx-data1)
-      (m/? (helper/new-task--wait-all-client-ops-sent))
-      (m/? (helper/new-task--client1-sync-barrier-1->2 "step6"))
-      (m/? helper/new-task--stop-rtc)
-      (helper/transact! conn tx-data2)
-      (let [r (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))]
-        (is (nil? r))
-        (m/? (helper/new-task--wait-all-client-ops-sent)))))
+     (let [conn (helper/get-downloaded-test-conn)
+           tx-data1 (const/tx-data-map :step6-delete-blocks-client1-1)
+           tx-data2 (const/tx-data-map :step6-delete-blocks-client1-2)]
+       (helper/transact! conn tx-data1)
+       (m/? (helper/new-task--wait-all-client-ops-sent))
+       (m/? (helper/new-task--client1-sync-barrier-1->2 "step6"))
+       (m/? helper/new-task--stop-rtc)
+       (helper/transact! conn tx-data2)
+       (let [r (m/? (rtc-core/new-task--rtc-start const/downloaded-test-repo const/test-token))]
+         (is (nil? r))
+         (m/? (helper/new-task--wait-all-client-ops-sent)))))
    :client2
    (m/sp
-    (let [conn (helper/get-downloaded-test-conn)]
-      (m/? (helper/new-task--client2-sync-barrier-1->2 "step6"))
-      (m/?
-       (c.m/backoff
-        (take 4 c.m/delays)
-        (m/sp
-         (let [page (d/pull @conn '[*] [:block/uuid const/step6-page-uuid])
-               page-blocks (when-let [page-id (:db/id page)]
-                             (ldb/get-page-blocks @conn page-id
-                                                  :pull-keys '[:block/uuid {:block/parent [:block/uuid]}]))]
-           (when-not (= 1 (count page-blocks))
-             (throw (ex-info "wait delete-blocks changes synced"
-                             {:missionary/retry true
-                              :page-blocks page-blocks})))
-           (is (= {:block/uuid const/step6-block3-uuid
-                   :block/parent {:block/uuid const/step6-page-uuid}}
-                  (select-keys (first page-blocks) [:block/uuid :block/parent])))))))))})
+     (let [conn (helper/get-downloaded-test-conn)]
+       (m/? (helper/new-task--client2-sync-barrier-1->2 "step6"))
+       (m/?
+        (c.m/backoff
+         (take 4 c.m/delays)
+         (m/sp
+           (let [page (d/pull @conn '[*] [:block/uuid const/step6-page-uuid])
+                 page-blocks (when-let [page-id (:db/id page)]
+                               (ldb/get-page-blocks @conn page-id
+                                                    :pull-keys '[:block/uuid {:block/parent [:block/uuid]}]))]
+             (when-not (= 1 (count page-blocks))
+               (throw (ex-info "wait delete-blocks changes synced"
+                               {:missionary/retry true
+                                :page-blocks page-blocks})))
+             (is (= {:block/uuid const/step6-block3-uuid
+                     :block/parent {:block/uuid const/step6-page-uuid}}
+                    (select-keys (first page-blocks) [:block/uuid :block/parent])))))))))})
 
 (defn- wrap-print-step-info
   [steps client]
   (map-indexed
    (fn [idx step]
      (m/sp
-      (helper/log "start step" idx)
-      (some-> (get step client) m/?)
-      (helper/log "end step" idx)))
+       (helper/log "start step" idx)
+       (some-> (get step client) m/?)
+       (helper/log "end step" idx)))
    steps))
 
 (def ^:private all-steps [step0 step1 step2 step3 step4 step5 step6])

src/rtc_e2e_test/fixture.cljs

@@ -3,11 +3,11 @@
             [const]
             [datascript.core :as d]
             [example]
-            [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.db-listener]
             [frontend.worker.state :as worker-state]
             [helper]
+            [frontend.common.missionary :as c.m]
             [missionary.core :as m]))
 
 (def install-some-consts

src/rtc_e2e_test/helper.cljs

@@ -1,13 +1,13 @@
 (ns helper
   (:require [cljs.test :as t :refer [is]]
-            [datascript.transit :as dt]
             [const]
             [datascript.core :as d]
-            [frontend.common.missionary-util :as c.m]
+            [datascript.transit :as dt]
             [frontend.worker.rtc.client-op :as client-op]
             [frontend.worker.rtc.core :as rtc.core]
             [frontend.worker.rtc.log-and-state :as rtc-log-and-state]
             [frontend.worker.state :as worker-state]
+            [frontend.common.missionary :as c.m]
             [logseq.db :as ldb]
             [logseq.db.frontend.order :as db-order]
             [logseq.outliner.batch-tx :as batch-tx]
@@ -91,11 +91,11 @@
   #_:clj-kondo/ignore
   (me/find
    client-op
-    [?op-type _ {:block-uuid ?block-uuid :av-coll [[!a !v _ !add] ...]}]
-    [?op-type ?block-uuid (map vector !a !v !add)]
+   [?op-type _ {:block-uuid ?block-uuid :av-coll [[!a !v _ !add] ...]}]
+   [?op-type ?block-uuid (map vector !a !v !add)]
 
-    [?op-type _ {:block-uuid ?block-uuid}]
-    [?op-type ?block-uuid]))
+   [?op-type _ {:block-uuid ?block-uuid}]
+   [?op-type ?block-uuid]))
 
 (defn new-task--wait-all-client-ops-sent
   [& {:keys [timeout] :or {timeout 10000}}]

src/test/frontend/test/helper.cljs

@@ -1,23 +1,24 @@
 (ns frontend.test.helper
   "Common helper fns for tests"
-  (:require [frontend.handler.file-based.repo :as file-repo-handler]
-            [frontend.state :as state]
-            [frontend.db.conn :as conn]
-            [clojure.string :as string]
-            [logseq.db.sqlite.util :as sqlite-util]
-            [frontend.db :as db]
-            [frontend.handler.editor :as editor-handler]
-            [frontend.handler.db-based.page :as db-page-handler]
+  (:require [clojure.string :as string]
             [datascript.core :as d]
-            [logseq.graph-parser.text :as text]
-            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
+            [frontend.background-tasks]
             [frontend.config :as config]
+            [frontend.db :as db]
+            [frontend.db.conn :as conn]
+            [frontend.handler.db-based.page :as db-page-handler]
+            [frontend.handler.editor :as editor-handler]
+            [frontend.handler.file-based.repo :as file-repo-handler]
+            [frontend.handler.file-based.status :as status]
+            [frontend.state :as state]
+            [frontend.worker.handler.page :as worker-page]
             [frontend.worker.pipeline :as worker-pipeline]
             [logseq.db.frontend.order :as db-order]
             [logseq.db.sqlite.build :as sqlite-build]
-            [frontend.handler.file-based.status :as status]
-            [logseq.outliner.db-pipeline :as db-pipeline]
-            [frontend.worker.handler.page :as worker-page]))
+            [logseq.db.sqlite.create-graph :as sqlite-create-graph]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.graph-parser.text :as text]
+            [logseq.outliner.db-pipeline :as db-pipeline]))
 
 (def node? (exists? js/process))