refactor(rtc,wip): re-implement update-op part in rtc,

use avt-coll data in update-op to simplify the code.
Handle different attributes in a unified way, instead of
dealing with each different attribute separately

.carve/ignore

@@ -79,7 +79,7 @@ frontend.db.model/get-all-classes
 ;; Initial loaded
 frontend.ui/_emoji-init-data
 ;; placeholder var for defonce
-frontend.worker.rtc.op-mem-layer/_sync-loop
+frontend.worker.rtc.op-mem-layer/_sync-loop-canceler
 ;; Used by shadow.cljs
 frontend.db-worker/init
 ;; WIP fn, remove when it's ready

scripts/src/logseq/tasks/dev.clj

@@ -38,7 +38,7 @@
   pass args through to cmd 'yarn cljs:run-test'"
   []
   (lint)
-  (test "-e" "long"))
+  (test "-e" "long" "-e" "fix-me"))
 
 
 (defn gen-malli-kondo-config

src/main/frontend/db_worker.cljs

@@ -172,7 +172,7 @@
       (let [schema (sqlite-util/get-schema repo)
             conn (sqlite-common-db/get-storage-conn storage schema)]
         (swap! *datascript-conns assoc repo conn)
-        (p/let [_ (op-mem-layer/<init-load-from-indexeddb! repo)]
+        (p/let [_ (op-mem-layer/<init-load-from-indexeddb2! repo)]
           (db-listener/listen-db-changes! repo conn))))))
 
 (defn- iter->vec [iter]

src/main/frontend/handler/worker.cljs

@@ -55,8 +55,9 @@
               ;; https://github.com/GoogleChromeLabs/comlink/blob/dffe9050f63b1b39f30213adeb1dd4b9ed7d2594/src/comlink.ts#L223-L236
               (if (and (= "HANDLER" (.-type data)) (= "throw" (.-name data)))
                 (if (.-isError (.-value data))
-                  (js/console.error "Unexpected webworker error:" (-> data bean/->clj (get-in [:value :value])))
-                  (js/console.error "Unexpected webworker error:" data))
+                  (do (js/console.error "Unexpected webworker error:" (-> data bean/->clj (get-in [:value :value])))
+                      (js/console.log (get-in (bean/->clj data) [:value :value :stack])))
+                  (js/console.error "Unexpected webworker error :" data))
                 (if (string? data)
                   (let [[e payload] (ldb/read-transit-str data)]
                     (handle (keyword e) wrapped-worker payload))

src/main/frontend/worker/db_listener.cljs

@@ -31,6 +31,14 @@
          (assoc m a datom))))
    {} entity-datoms))
 
+(defn- entity-datoms=>a->add?->v->t
+  [entity-datoms]
+  (reduce
+   (fn [m datom]
+     (let [[_e a v t add?] datom]
+       (assoc-in m [a add? v] t)))
+   {} entity-datoms))
+
 
 (defmulti listen-db-changes
   (fn [listen-key & _] listen-key))
@@ -103,10 +111,14 @@ generate undo ops.")
                              id-order (distinct (map first datom-vec-coll))
                              same-entity-datoms-coll (map id->same-entity-datoms id-order)
                              id->attr->datom (update-vals id->same-entity-datoms entity-datoms=>attr->datom)
+                             e->a->add?->v->t (update-vals
+                                               id->same-entity-datoms
+                                               entity-datoms=>a->add?->v->t)
                              args* (assoc tx-report
                                           :repo repo
                                           :conn conn
                                           :id->attr->datom id->attr->datom
+                                          :e->a->add?->v->t e->a->add?->v->t
                                           :same-entity-datoms-coll same-entity-datoms-coll)]
                          (doseq [[k handler-fn] handlers]
                            (handler-fn k args*))))))))))

src/main/frontend/worker/rtc/client.cljs

@@ -1,7 +1,6 @@
 (ns frontend.worker.rtc.client
   "Fns about push local updates"
   (:require [clojure.set :as set]
-            [cognitect.transit :as transit]
             [datascript.core :as d]
             [frontend.common.missionary-util :as c.m]
             [frontend.worker.rtc.const :as rtc-const]
@@ -12,8 +11,6 @@
             [logseq.db :as ldb]
             [missionary.core :as m]))
 
-(def ^:private transit-w (transit/writer :json))
-
 (defn- handle-remote-ex
   [resp]
   (if-let [e ({:graph-not-exist r.ex/ex-remote-graph-not-exist
@@ -32,13 +29,13 @@
 (defn- register-graph-updates
   [get-ws-create-task graph-uuid]
   (m/sp
-   (try
-     (m/? (send&recv get-ws-create-task {:action "register-graph-updates"
-                                         :graph-uuid graph-uuid}))
-     (catch :default e
-       (if (= :rtc.exception/remote-graph-not-ready (:type (ex-data e)))
-         (throw (ex-info "remote graph is still creating" {:missionary/retry true} e))
-         (throw e))))))
+    (try
+      (m/? (send&recv get-ws-create-task {:action "register-graph-updates"
+                                          :graph-uuid graph-uuid}))
+      (catch :default e
+        (if (= :rtc.exception/remote-graph-not-ready (:type (ex-data e)))
+          (throw (ex-info "remote graph is still creating" {:missionary/retry true} e))
+          (throw e))))))
 
 (defn- ensure-register-graph-updates*
   "Return a task: get or create a mws(missionary wrapped websocket).
@@ -61,23 +58,13 @@
 
 (def ensure-register-graph-updates (memoize ensure-register-graph-updates*))
 
-(defn- remove-non-exist-block-uuids-in-add-retract-map
-  [conn add-retract-map]
-  (let [{:keys [add retract]} add-retract-map
-        add* (->> add
-                  (map (fn [x] [:block/uuid x]))
-                  (d/pull-many @conn [:block/uuid])
-                  (keep :block/uuid))]
-    (cond-> {}
-      (seq add*) (assoc :add add*)
-      (seq retract) (assoc :retract retract))))
-
 (defn- ->pos
   [left-uuid parent-uuid]
-  (cond
-    (or (nil? left-uuid) (nil? parent-uuid)) :no-order
-    (not= left-uuid parent-uuid)             :sibling
-    :else                                    :child))
+  (case [(some? left-uuid) (some? parent-uuid)]
+    [true true]   [left-uuid (if (not= left-uuid parent-uuid) :sibling :child)]
+    [true false]  [left-uuid :no-parent-sibling]
+    [false true]  [parent-uuid :no-order]
+    [false false] [nil :no-parent-sibling]))
 
 (defmulti ^:private local-block-ops->remote-ops-aux (fn [tp & _] tp))
 
@@ -86,57 +73,51 @@
   (when parent-uuid
     (let [target-uuid (or left-uuid parent-uuid)
           pos         (->pos left-uuid parent-uuid)]
-      (swap! *remote-ops conj [:move {:block-uuid block-uuid :target-uuid target-uuid :pos pos}])
-      (swap! *depend-on-block-uuid-set conj target-uuid))))
+      (swap! *remote-ops conj [:move {:block-uuid block-uuid :pos pos}])
+      (when target-uuid
+        (swap! *depend-on-block-uuid-set conj target-uuid)))))
+
+(defn- card-many-attr?
+  [db attr]
+  (= :db.cardinality/many (get-in (d/schema db) [attr :db/cardinality])))
+
+
+(defn- remove-redundant-av
+  "Remove previous av if later-av has same [a v] or a"
+  [db av-coll]
+  (loop [[av & others] av-coll
+         r {} ;; [a v] or `a` -> [a v t add?]
+              ;; [a v] as key for card-many attr, `a` as key for card-one attr
+         ]
+    (if-not av
+      (sort-by #(nth % 2) (vals r))
+      (let [[a v _t _add?] av
+            av-key (if (card-many-attr? db a) [a v] a)]
+        (recur others (assoc r av-key av))))))
+
+(defn- remove-non-exist-ref-av
+  "Remove av if its v is ref(block-uuid) and not exist"
+  [db av-coll]
+  (remove
+   (fn [av]
+     (let [[_a v] av]
+       (and (uuid? v)
+            (nil? (d/entity db [:block/uuid v])))))
+   av-coll))
 
 (defmethod local-block-ops->remote-ops-aux :update-op
-  [_ & {:keys [conn user-uuid block update-op left-uuid parent-uuid *remote-ops]}]
-  (let [block-uuid          (:block/uuid block)
-        attr-map            (:updated-attrs (second update-op))
-        attr-alias-map      (when (contains? attr-map :alias)
-                              (remove-non-exist-block-uuids-in-add-retract-map conn (:alias attr-map)))
-        attr-tags-map       (when (contains? attr-map :tags)
-                              (remove-non-exist-block-uuids-in-add-retract-map conn (:tags attr-map)))
-        attr-type-map       (when (contains? attr-map :type)
-                              (let [{:keys [add retract]} (:type attr-map)
-                                    current-type-value    (set (:block/type block))
-                                    add                   (set/intersection add current-type-value)
-                                    retract               (set/difference retract current-type-value)]
-                                (cond-> {}
-                                  (seq add)     (assoc :add add)
-                                  (seq retract) (assoc :retract retract))))
-        attr-properties-map (when (contains? attr-map :properties)
-                              (let [{:keys [add retract]} (:properties attr-map)
-                                    properties            (:block/properties block)
-                                    add*                  (into []
-                                                                (update-vals (select-keys properties add)
-                                                                             (partial transit/write transit-w)))]
-                                (cond-> {}
-                                  (seq add*)    (assoc :add add*)
-                                  (seq retract) (assoc :retract retract))))
-        target-uuid         (or left-uuid parent-uuid)
-        pos                 (->pos left-uuid parent-uuid)]
+  [_ & {:keys [db block update-op left-uuid parent-uuid *remote-ops *depend-on-block-uuid-set]}]
+  (let [block-uuid (:block/uuid block)
+        pos (->pos left-uuid parent-uuid)
+        av-coll (->> (:av-coll (last update-op))
+                     (remove-redundant-av db)
+                     (remove-non-exist-ref-av db))
+        depend-on-block-uuids (keep (fn [[_a v]] (when (uuid? v) v)) av-coll)]
     (swap! *remote-ops conj
-           [:update
-            (cond-> {:block-uuid block-uuid}
-              (:block/journal-day block)    (assoc :journal-day (:block/journal-day block))
-              (:block/updated-at block)     (assoc :updated-at (:block/updated-at block))
-              (:block/created-at block)     (assoc :created-at (:block/created-at block))
-              (= (:block/updated-at block)
-                 (:block/created-at block)) (assoc :created-by user-uuid)
-              (contains? attr-map :schema)  (assoc :schema
-                                                   (transit/write transit-w (:block/schema block)))
-              attr-alias-map                (assoc :alias attr-alias-map)
-              attr-type-map                 (assoc :type attr-type-map)
-              attr-tags-map                 (assoc :tags attr-tags-map)
-              attr-properties-map           (assoc :properties attr-properties-map)
-              (and (contains? attr-map :content)
-                   (:block/raw-content block))
-              (assoc :content (:block/raw-content block))
-              (and (contains? attr-map :link)
-                   (:block/uuid (:block/link block)))
-              (assoc :link (:block/uuid (:block/link block)))
-              target-uuid                   (assoc :target-uuid target-uuid :pos pos))])))
+           [:update {:block-uuid block-uuid
+                     :pos pos
+                     :av-coll av-coll}])
+    (swap! *depend-on-block-uuid-set (partial apply conj) depend-on-block-uuids)))
 
 (defmethod local-block-ops->remote-ops-aux :update-page-op
   [_ & {:keys [conn block-uuid *remote-ops]}]
@@ -149,24 +130,24 @@
 
 (defmethod local-block-ops->remote-ops-aux :remove-op
   [_ & {:keys [conn remove-op *remote-ops]}]
-  (when-let [block-uuid (:block-uuid (second remove-op))]
+  (when-let [block-uuid (:block-uuid (last remove-op))]
     (when (nil? (d/entity @conn [:block/uuid block-uuid]))
       (swap! *remote-ops conj [:remove {:block-uuids [block-uuid]}]))))
 
 (defmethod local-block-ops->remote-ops-aux :remove-page-op
   [_ & {:keys [conn remove-page-op *remote-ops]}]
-  (when-let [block-uuid (:block-uuid (second remove-page-op))]
+  (when-let [block-uuid (:block-uuid (last remove-page-op))]
     (when (nil? (d/entity @conn [:block/uuid block-uuid]))
       (swap! *remote-ops conj [:remove-page {:block-uuid block-uuid}]))))
 
 (defn- local-block-ops->remote-ops
-  [repo conn user-uuid block-ops]
+  [repo conn block-ops]
   (let [*depend-on-block-uuid-set (atom #{})
         *remote-ops (atom [])
         {move-op :move remove-op :remove update-op :update update-page-op :update-page remove-page-op :remove-page}
         block-ops]
     (when-let [block-uuid
-               (some (comp :block-uuid second) [move-op update-op update-page-op])]
+               (some (comp :block-uuid last) [move-op update-op update-page-op])]
       (when-let [block (d/entity @conn [:block/uuid block-uuid])]
         (let [left-uuid (:block/uuid (ldb/get-left-sibling block))
               parent-uuid (some-> block :block/parent :block/uuid)]
@@ -182,15 +163,13 @@
           ;; remote-update-op
           (when update-op
             (local-block-ops->remote-ops-aux :update-op
-                                             :repo repo
-                                             :user-uuid user-uuid
-                                             :conn conn
+                                             :db @conn
                                              :block block
                                              :update-op update-op
                                              :parent-uuid parent-uuid
                                              :left-uuid left-uuid
                                              :*remote-ops *remote-ops
-                                             :created-by user-uuid)))
+                                             :*depend-on-block-uuid-set *depend-on-block-uuid-set)))
         ;; remote-update-page-op
         (when update-page-op
           (local-block-ops->remote-ops-aux :update-page-op
@@ -218,7 +197,7 @@
      :depend-on-block-uuids @*depend-on-block-uuid-set}))
 
 (defn- gen-block-uuid->remote-ops
-  [repo conn user-uuid & {:keys [n] :or {n 50}}]
+  [repo conn & {:keys [n] :or {n 50}}]
   (loop [current-handling-block-ops nil
          current-handling-block-uuid nil
          depend-on-block-uuid-coll nil
@@ -231,10 +210,10 @@
 
       (and (empty? current-handling-block-ops)
            (empty? depend-on-block-uuid-coll))
-      (if-let [{min-epoch-block-ops :ops block-uuid :block-uuid} (op-mem-layer/get-min-epoch-block-ops repo)]
+      (if-let [{min-t-block-ops :ops block-uuid :block-uuid} (op-mem-layer/get-min-t-block-ops repo)]
         (do (assert (not (contains? r block-uuid)) {:r r :block-uuid block-uuid})
             (op-mem-layer/remove-block-ops! repo block-uuid)
-            (recur min-epoch-block-ops block-uuid depend-on-block-uuid-coll r))
+            (recur min-t-block-ops block-uuid depend-on-block-uuid-coll r))
         ;; finish
         r)
 
@@ -247,7 +226,7 @@
 
       (seq current-handling-block-ops)
       (let [{:keys [remote-ops depend-on-block-uuids]}
-            (local-block-ops->remote-ops repo conn user-uuid current-handling-block-ops)]
+            (local-block-ops->remote-ops repo conn current-handling-block-ops)]
         (recur nil nil
                (set/union (set depend-on-block-uuid-coll)
                           (op-mem-layer/intersection-block-uuids repo depend-on-block-uuids))
@@ -270,6 +249,7 @@
                         [block-uuid (:target-uuid move-op)])))
               block-uuid->remote-ops)
         all-move-uuids (set (keys block-uuid->dep-uuid))
+        ;; TODO: use `sort-coll-by-dependency`
         sorted-uuids
         (loop [r []
                rest-uuids all-move-uuids
@@ -307,40 +287,42 @@
 
 (defn new-task--push-local-ops
   "Return a task: push local updates"
-  [repo conn user-uuid graph-uuid date-formatter get-ws-create-task add-log-fn]
+  [repo conn graph-uuid date-formatter get-ws-create-task add-log-fn]
   (m/sp
-    (when-let [ops-for-remote (rtc-const/to-ws-ops-decoder
-                               (sort-remote-ops
-                                (gen-block-uuid->remote-ops repo conn user-uuid)))]
-      (op-mem-layer/new-branch! repo)
-      (let [local-tx (op-mem-layer/get-local-tx repo)
-            r (m/? (send&recv get-ws-create-task {:action "apply-ops" :graph-uuid graph-uuid
-                                                  :ops ops-for-remote :t-before (or local-tx 1)}))]
-        (if-let [remote-ex (:ex-data r)]
-          (do (add-log-fn remote-ex)
-              (case (:type remote-ex)
-              ;; - :graph-lock-failed
-              ;;   conflict-update remote-graph, keep these local-pending-ops
-              ;;   and try to send ops later
-                :graph-lock-failed
-                (do (op-mem-layer/rollback! repo)
-                    nil)
-              ;; - :graph-lock-missing
-              ;;   this case means something wrong in remote-graph data,
-              ;;   nothing to do at client-side
-                :graph-lock-missing
-                (do (op-mem-layer/rollback! repo)
-                    (throw r.ex/ex-remote-graph-lock-missing))
+    (when-let [remote-ops (gen-block-uuid->remote-ops repo conn)]
+      (when-let [ops-for-remote (rtc-const/to-ws-ops-decoder
+                                 (sort-remote-ops
+                                  remote-ops))]
+        (assert (= (count remote-ops) (count ops-for-remote)))
+        (op-mem-layer/new-branch! repo)
+        (let [local-tx (op-mem-layer/get-local-tx repo)
+              r (m/? (send&recv get-ws-create-task {:action "apply-ops" :graph-uuid graph-uuid
+                                                    :ops ops-for-remote :t-before (or local-tx 1)}))]
+          (if-let [remote-ex (:ex-data r)]
+            (do (add-log-fn remote-ex)
+                (case (:type remote-ex)
+                  ;; - :graph-lock-failed
+                  ;;   conflict-update remote-graph, keep these local-pending-ops
+                  ;;   and try to send ops later
+                  :graph-lock-failed
+                  (do (op-mem-layer/rollback! repo)
+                      nil)
+                  ;; - :graph-lock-missing
+                  ;;   this case means something wrong in remote-graph data,
+                  ;;   nothing to do at client-side
+                  :graph-lock-missing
+                  (do (op-mem-layer/rollback! repo)
+                      (throw r.ex/ex-remote-graph-lock-missing))
 
-                :rtc.exception/get-s3-object-failed
-                (do (op-mem-layer/rollback! repo)
-                    nil)
-              ;; else
-                (do (op-mem-layer/rollback! repo)
-                    (throw (ex-info "Unavailable" {:remote-ex remote-ex})))))
+                  :rtc.exception/get-s3-object-failed
+                  (do (op-mem-layer/rollback! repo)
+                      nil)
+                  ;; else
+                  (do (op-mem-layer/rollback! repo)
+                      (throw (ex-info "Unavailable" {:remote-ex remote-ex})))))
 
-          (do (assert (pos? (:t r)) r)
-              (op-mem-layer/commit! repo)
-              (r.remote-update/apply-remote-update
-               repo conn date-formatter {:type :remote-update :value r} add-log-fn)
-              (add-log-fn {:type ::push-client-updates :remote-t (:t r)})))))))
+            (do (assert (pos? (:t r)) r)
+                (op-mem-layer/commit! repo)
+                (r.remote-update/apply-remote-update
+                 repo conn date-formatter {:type :remote-update :value r} add-log-fn)
+                (add-log-fn {:type ::push-client-updates :remote-t (:t r)}))))))))

src/main/frontend/worker/rtc/const.cljs

@@ -4,7 +4,6 @@
             [malli.core :as m]
             [malli.transform :as mt]))
 
-
 (def general-attrs-schema-coll
   [[:updated-at {:optional true} :int]
    [:created-at {:optional true} :int]
@@ -22,13 +21,85 @@
 (def general-attr-set
   (into #{} (map first) general-attrs-schema-coll))
 
-(def block-type-schema [:enum "property" "class" "whiteboard" "hidden" "closed value" "macro"])
+;; (def block-type-schema [:enum "property" "class" "whiteboard" "hidden" "closed value" "macro"])
+
+;; (def block-pos-type-schema
+;;   [:enum :sibling :child :no-order])
+
 (def block-pos-schema
   ":sibling:  sibling of target-block(:target-uuid)
   :child: child of target-block(:target-uuid)
-  :no-order: this block doesn't have :block/order attr"
-  [:enum :sibling :child :no-order])
+  :no-order: this block doesn't have :block/order attr
+  :no-parent-sibling: this block doesn't have :block/parent,
+                      and it's sibling of target-uuid(if nil, it's the first one)"
+  [:catn
+   [:target-uuid [:maybe :uuid]]
+   [:pos [:enum :sibling :child :no-order :no-parent-sibling]]])
+
+(comment
+  (def to-ws-op-schema-deprecated
+    "TODO: remove this schema"
+    [:multi {:dispatch first :decode/string #(update % 0 keyword)}
+     [:move
+      [:cat :keyword
+       [:map
+        [:block-uuid :uuid]
+        [:target-uuid :uuid]
+        [:pos block-pos-type-schema]]]]
+     [:remove
+      [:cat :keyword
+       [:map
+        [:block-uuids [:sequential :uuid]]]]]
+
+     [:update
+      [:cat :keyword
+       [:map
+        [:block-uuid :uuid]
+        [:target-uuid {:optional true} :uuid]
+        [:pos {:optional true} block-pos-type-schema]
+        [:content {:optional true} :string]
+        [:updated-at {:optional true} :int]
+        [:created-at {:optional true} :int]
+        [:created-by {:optional true} :string]
+        [:tags {:optional true} [:map
+                                 [:add {:optional true} [:maybe [:set :uuid]]]
+                                 [:retract {:optional true} [:maybe [:set :uuid]]]]]
+        [:alias {:optional true} [:map
+                                  [:add {:optional true} [:maybe [:set :uuid]]]
+                                  [:retract {:optional true} [:maybe [:set :uuid]]]]]
+        [:type {:optional true} [:map
+                                 [:add {:optional true} [:maybe [:set block-type-schema]]]
+                                 [:retract {:optional true} [:maybe [:set block-type-schema]]]]]
+        [:schema {:optional true} :string ;transit-string
+         ]
+        [:properties {:optional true} [:map
+                                       [:add {:optional true} [:sequential [:cat :uuid :string ;; transit-string
+                                                                            ]]]
+                                       [:retract {:optional true} [:set :uuid]]]]
+        [:link {:optional true} :uuid]
+        [:journal-day {:optional true} :int]
+        [:ident {:optional true} :string]]]]
+     [:update-page
+      [:cat :keyword
+       [:map
+        [:block-uuid :uuid]
+        [:page-name :string]
+        [:original-name :string]]]]
+     [:remove-page
+      [:cat :keyword
+       [:map
+        [:block-uuid :uuid]]]]]))
 
+(def av-schema
+  [:cat
+   :keyword
+   [:or
+    :uuid   ;; reference type
+    :string ;; all other type value convert to string by transit
+    ]
+   :int     ;; t
+   :boolean ;; add(true) or retract
+   ])
 
 (def to-ws-op-schema
   [:multi {:dispatch first :decode/string #(update % 0 keyword)}
@@ -36,41 +107,11 @@
     [:cat :keyword
      [:map
       [:block-uuid :uuid]
-      [:target-uuid :uuid]
       [:pos block-pos-schema]]]]
    [:remove
     [:cat :keyword
      [:map
       [:block-uuids [:sequential :uuid]]]]]
-
-   [:update
-    [:cat :keyword
-     [:map
-      [:block-uuid :uuid]
-      [:target-uuid {:optional true} :uuid]
-      [:pos {:optional true} block-pos-schema]
-      [:content {:optional true} :string]
-      [:updated-at {:optional true} :int]
-      [:created-at {:optional true} :int]
-      [:created-by {:optional true} :string]
-      [:tags {:optional true} [:map
-                               [:add {:optional true} [:maybe [:set :uuid]]]
-                               [:retract {:optional true} [:maybe [:set :uuid]]]]]
-      [:alias {:optional true} [:map
-                                [:add {:optional true} [:maybe [:set :uuid]]]
-                                [:retract {:optional true} [:maybe [:set :uuid]]]]]
-      [:type {:optional true} [:map
-                               [:add {:optional true} [:maybe [:set block-type-schema]]]
-                               [:retract {:optional true} [:maybe [:set block-type-schema]]]]]
-      [:schema {:optional true} :string ;transit-string
-       ]
-      [:properties {:optional true} [:map
-                                     [:add {:optional true} [:sequential [:cat :uuid :string ;; transit-string
-                                                                          ]]]
-                                     [:retract {:optional true} [:set :uuid]]]]
-      [:link {:optional true} :uuid]
-      [:journal-day {:optional true} :int]
-      [:ident {:optional true} :string]]]]
    [:update-page
     [:cat :keyword
      [:map
@@ -80,7 +121,13 @@
    [:remove-page
     [:cat :keyword
      [:map
-      [:block-uuid :uuid]]]]])
+      [:block-uuid :uuid]]]]
+   [:update
+    [:cat :keyword
+     [:map
+      [:block-uuid :uuid]
+      [:pos block-pos-schema]
+      [:av-coll [:sequential av-schema]]]]]])
 
 (def to-ws-ops-validator (m/validator [:sequential to-ws-op-schema]))
 (def to-ws-ops-decoder (m/decoder [:sequential to-ws-op-schema] mt/string-transformer))

src/main/frontend/worker/rtc/core2.cljs

@@ -131,7 +131,7 @@
 (defn- create-rtc-loop
   "Return a map with [:rtc-log-flow :rtc-state-flow :rtc-loop-task :*rtc-auto-push? :onstarted-task]
   TODO: auto refresh token if needed"
-  [user-uuid graph-uuid repo conn date-formatter token
+  [graph-uuid repo conn date-formatter token
    & {:keys [auto-push? debug-ws-url] :or {auto-push? true}}]
   (let [ws-url              (or debug-ws-url (get-ws-url token))
         *auto-push?         (atom auto-push?)
@@ -163,7 +163,7 @@
 
                :local-update-check
                (m/? (r.client/new-task--push-local-ops
-                     repo conn user-uuid graph-uuid date-formatter
+                     repo conn graph-uuid date-formatter
                      get-ws-create-task add-log-fn))))
            (m/ap)
            (m/reduce {} nil)
@@ -192,7 +192,7 @@
               config (worker-state/get-config repo)
               date-formatter (common-config/get-date-formatter config)
               {:keys [onstarted-task rtc-log-flow rtc-state-flow *rtc-auto-push? rtc-loop-task]}
-              (create-rtc-loop user-uuid graph-uuid repo conn date-formatter token)
+              (create-rtc-loop graph-uuid repo conn date-formatter token)
               canceler (c.m/run-task rtc-loop-task :rtc-loop-task)
               start-ex (m/? onstarted-task)]
           (if (:ex-data start-ex)

src/main/frontend/worker/rtc/db_listener.cljs

@@ -1,131 +1,108 @@
 (ns frontend.worker.rtc.db-listener
   "listen datascript changes, infer operations from the db tx-report"
-  (:require [cljs-time.coerce :as tc]
-            [cljs-time.core :as t]
-            [clojure.data :as data]
-            [clojure.set :as set]
-            [datascript.core :as d]
+  (:require [datascript.core :as d]
             [frontend.schema-register :include-macros true :as sr]
             [frontend.worker.db-listener :as db-listener]
-            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]))
-
-
-(defn- diff-value-of-set-type-attr
-  [db-before db-after eid attr-name]
-  (let [current-value-set (set (get (d/entity db-after eid) attr-name))
-        old-value-set (set (get (d/entity db-before eid) attr-name))
-        add (set/difference current-value-set old-value-set)
-        retract (set/difference old-value-set current-value-set)]
-    (cond-> {}
-      (seq add) (conj [:add add])
-      (seq retract) (conj [:retract retract]))))
-
-
-(defn- diff-properties-value
-  [db-before db-after eid]
-  (let [current-value (:block/properties (d/entity db-after eid))
-        old-value (:block/properties (d/entity db-before eid))
-        [only-in-current-uuids only-in-old-uuids both-uuids] (map (comp set keys) (data/diff current-value old-value))
-        add-uuids only-in-current-uuids
-        retract-uuids (set/difference only-in-old-uuids add-uuids both-uuids)
-        update-uuids (set/intersection both-uuids only-in-old-uuids)
-        add-uuids* (set/union add-uuids update-uuids)]
-    (cond-> {}
-      (seq add-uuids*) (conj [:add add-uuids*])
-      (seq retract-uuids) (conj [:retract retract-uuids]))))
-
-(defn- entity-datoms=>ops
-  [db-before db-after id->attr->datom entity-datoms]
-  (let [e (ffirst entity-datoms)
-        attr->datom (id->attr->datom e)]
-    (when (seq attr->datom)
-      (let [updated-key-set (set (keys attr->datom))
-            {[_e _a block-uuid _t add1?] :block/uuid
-             [_e _a _v _t add2?]         :block/name
-             [_e _a _v _t add3?]         :block/parent
-             [_e _a _v _t add4?]         :block/order
-             [_e _a _v _t add5?]         :block/original-name} attr->datom
-            ops (cond
-                  (and (not add1?) block-uuid
-                       (not add2?) (contains? updated-key-set :block/name))
-                  [[:remove-page block-uuid]]
-
-                  (and (not add1?) block-uuid)
-                  [[:remove block-uuid]]
-
-                  :else
-                  (let [updated-general-attrs (seq (set/intersection
-                                                    updated-key-set
-                                                    #{:block/tags :block/alias :block/type :block/schema :block/content
-                                                      :block/properties :block/link :block/journal-day}))
-                        ops (cond-> []
-                              (or add3? add4?)
-                              (conj [:move])
-
-                              (or (and (contains? updated-key-set :block/name) add2?)
-                                  (and (contains? updated-key-set :block/original-name) add5?))
-                              (conj [:update-page]))
-                        update-op (->> updated-general-attrs
-                                       (keep
-                                        (fn [attr-name]
-                                          (case attr-name
-                                            (:block/link :block/schema :block/content :block/journal-day)
-                                            {(keyword (name attr-name)) nil}
-
-                                            :block/alias
-                                            (let [diff-value (diff-value-of-set-type-attr db-before db-after e :block/alias)]
-                                              (when (seq diff-value)
-                                                (let [{:keys [add retract]} diff-value
-                                                      add (keep :block/uuid (d/pull-many db-after [:block/uuid]
-                                                                                         (map :db/id add)))
-                                                      retract (keep :block/uuid (d/pull-many db-before [:block/uuid]
-                                                                                             (map :db/id retract)))]
-                                                  {:alias (cond-> {}
-                                                            (seq add) (conj [:add add])
-                                                            (seq retract) (conj [:retract retract]))})))
-
-                                            :block/type
-                                            (let [diff-value (diff-value-of-set-type-attr db-before db-after e :block/type)]
-                                              (when (seq diff-value)
-                                                {:type diff-value}))
-
-                                            :block/tags
-                                            (let [diff-value (diff-value-of-set-type-attr db-before db-after e :block/tags)]
-                                              (when (seq diff-value)
-                                                (let [{:keys [add retract]} diff-value
-                                                      add (keep :block/uuid (d/pull-many db-after [:block/uuid]
-                                                                                         (map :db/id add)))
-                                                      retract (keep :block/uuid (d/pull-many db-before [:block/uuid]
-                                                                                             (map :db/id retract)))]
-                                                  {:tags (cond-> {}
-                                                           (seq add) (conj [:add add])
-                                                           (seq retract) (conj [:retract retract]))})))
-                                            :block/properties
-                                            (let [diff-value (diff-properties-value db-before db-after e)]
-                                              (when (seq diff-value)
-                                                (let [{:keys [add retract]} diff-value
-                                                      add (keep :block/uuid (d/pull-many
-                                                                             db-after [:block/uuid]
-                                                                             (map (fn [uuid] [:block/uuid uuid]) add)))]
-                                                  {:properties (cond-> {}
-                                                                 (seq add) (conj [:add add])
-                                                                 (seq retract) (conj [:retract retract]))})))
-                                            ;; else
-                                            nil)))
-                                       (apply merge))]
-                    (cond-> ops (seq update-op) (conj [:update update-op]))))
-            ops* (keep (fn [op]
-                         (let [block-uuid (some-> (d/entity db-after e) :block/uuid str)]
-                           (case (first op)
-                             :move        (when block-uuid ["move" {:block-uuid block-uuid}])
-                             :update      (when block-uuid
-                                            ["update" (cond-> {:block-uuid block-uuid}
-                                                        (second op) (conj [:updated-attrs (second op)]))])
-                             :update-page (when block-uuid ["update-page" {:block-uuid block-uuid}])
-                             :remove      ["remove" {:block-uuid (str (second op))}]
-                             :remove-page ["remove-page" {:block-uuid (str (second op))}])))
-                       ops)]
-        ops*))))
+            [frontend.worker.rtc.op-mem-layer :as op-mem-layer]
+            [logseq.db :as ldb]))
+
+(defn- latest-add?->v->t
+  [add?->v->t]
+  (let [latest-add     (first (sort-by second > (seq (add?->v->t true))))
+        latest-retract (first (sort-by second > (seq (add?->v->t false))))]
+    (cond
+      (nil? latest-add)                               {false (conj {} latest-retract)}
+      (nil? latest-retract)                           {true (conj {} latest-add)}
+      (= (second latest-add) (second latest-retract)) {true (conj {} latest-add)
+                                                       false (conj {} latest-retract)}
+      (> (second latest-add) (second latest-retract)) {true (conj {} latest-add)}
+      :else                                           {false (conj {} latest-retract)})))
+
+(def ^:private const-concerned-attrs
+  #{:block/content :block/created-at :block/updated-at :block/alias
+    :block/tags :block/type :block/schema :block/link :block/journal-day})
+
+(defn- concerned-attr?
+  [attr]
+  (contains? const-concerned-attrs attr))
+
+(defn- ref-attr?
+  [db attr]
+  (= :db.type/ref (get-in (d/schema db) [attr :db/valueType])))
+
+;; (defn- card-many-attr?
+;;   [db attr]
+;;   (= :db.cardinality/many (get-in (d/schema db) [attr :db/cardinality])))
+
+(defn- update-op-av-coll
+  [db-before db-after a->add?->v->t]
+  (mapcat
+   (fn [[a add?->v->t]]
+     (mapcat
+      (fn [[add? v->t]]
+        (keep
+         (fn [[v t]]
+           (let [ref? (ref-attr? db-after a)]
+             (case [add? ref?]
+               [true true]
+               (when-let [v-uuid (:block/uuid (d/entity db-after v))]
+                 [a v-uuid t add?])
+               [false true]
+               (when-let [v-uuid (:block/uuid
+                                  (or (d/entity db-after v)
+                                      (d/entity db-before v)))]
+                 [a v-uuid t add?])
+               ([true false] [false false]) [a (ldb/write-transit-str v) t add?])))
+         v->t))
+      add?->v->t))
+   a->add?->v->t))
+
+(defn max-t
+  [a->add?->v->t]
+  (apply max (mapcat vals (mapcat vals (vals a->add?->v->t)))))
+
+(defn- get-first-vt
+  [add?->v->t k]
+  (some-> add?->v->t (get k) first))
+
+(defn- entity-datoms=>ops2
+  [db-before db-after e->a->add?->v->t entity-datoms]
+  (let [e                            (ffirst entity-datoms)
+        block-uuid                   (:block/uuid (d/entity db-after e))
+        a->add?->v->t                (e->a->add?->v->t e)
+        {add?->block-name->t          :block/name
+         add?->block-original-name->t :block/original-name
+         add?->block-uuid->t          :block/uuid
+         add?->block-parent->t        :block/parent
+         add?->block-order->t         :block/order}
+        a->add?->v->t
+        [retract-block-uuid t1]      (some-> add?->block-uuid->t (get false) first)
+        [retract-block-name _]       (some-> add?->block-name->t (get false) first)
+        [add-block-name t2]          (some-> add?->block-name->t latest-add?->v->t (get-first-vt true))
+        [add-block-original-name t3] (some-> add?->block-original-name->t
+                                             latest-add?->v->t
+                                             (get-first-vt true))
+        [add-block-parent t4]        (some-> add?->block-parent->t latest-add?->v->t (get-first-vt true))
+        [add-block-order t5]         (some-> add?->block-order->t latest-add?->v->t (get-first-vt true))
+        a->add?->v->t*               (into {} (filter (fn [[a _]] (concerned-attr? a)) a->add?->v->t))]
+    (cond
+      (and retract-block-uuid retract-block-name)
+      [[:remove-page t1 {:block-uuid retract-block-uuid}]]
+
+      retract-block-uuid
+      [[:remove t1 {:block-uuid retract-block-uuid}]]
+
+      :else
+      (let [ops (cond-> []
+                  (or add-block-parent add-block-order)
+                  (conj [:move (or t4 t5) {:block-uuid block-uuid}])
+
+                  (or add-block-name add-block-original-name)
+                  (conj [:update-page (or t2 t3) {:block-uuid block-uuid}]))
+            update-op (when-let [av-coll (not-empty (update-op-av-coll db-before db-after a->add?->v->t*))]
+                        (let [t (max-t a->add?->v->t*)]
+                          [:update t {:block-uuid block-uuid :av-coll av-coll}]))]
+        (cond-> ops update-op (conj update-op))))))
 
 (defn- entity-datoms=>asset-op
   [db-after id->attr->datom entity-datoms]
@@ -148,30 +125,21 @@
                 :update-asset (when asset-uuid ["update-asset" {:asset-uuid asset-uuid}])
                 :remove-asset ["remove-asset" {:asset-uuid (str (second op))}]))))))))
 
-
 (defn generate-rtc-ops
-  [repo db-before db-after same-entity-datoms-coll id->attr->datom]
+  [repo db-before db-after same-entity-datoms-coll id->attr->datom e->a->v->add?->t]
   (let [asset-ops (keep (partial entity-datoms=>asset-op db-after id->attr->datom) same-entity-datoms-coll)
         ops (when (empty asset-ops)
-              (mapcat (partial entity-datoms=>ops db-before db-after id->attr->datom) same-entity-datoms-coll))
-        now-epoch*1000 (* 1000 (tc/to-long (t/now)))
-        ops* (map-indexed (fn [idx op]
-                            [(first op) (assoc (second op) :epoch (+ idx now-epoch*1000))]) ops)
-        epoch2 (+ now-epoch*1000 (count ops))
-        asset-ops* (map-indexed (fn [idx op]
-                                  [(first op) (assoc (second op) :epoch (+ idx epoch2))]) asset-ops)]
-    (when (seq ops*)
-      (op-mem-layer/add-ops! repo ops*))
-    (when (seq asset-ops*)
-      (op-mem-layer/add-asset-ops! repo asset-ops*))))
-
+              (mapcat (partial entity-datoms=>ops2 db-before db-after e->a->v->add?->t)
+                      same-entity-datoms-coll))]
+    (when (seq ops)
+      (op-mem-layer/add-ops! repo ops))))
 
 (sr/defkeyword :persist-op?
   "tx-meta option, generate rtc ops when not nil (default true)")
 
 (defmethod db-listener/listen-db-changes :gen-rtc-ops
   [_ {:keys [_tx-data tx-meta db-before db-after
-             repo id->attr->datom same-entity-datoms-coll]}]
+             repo id->attr->datom e->a->add?->v->t same-entity-datoms-coll]}]
   (when (and (op-mem-layer/rtc-db-graph? repo)
              (:persist-op? tx-meta true))
-    (generate-rtc-ops repo db-before db-after same-entity-datoms-coll id->attr->datom)))
+    (generate-rtc-ops repo db-before db-after same-entity-datoms-coll id->attr->datom e->a->add?->v->t)))

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -64,7 +64,7 @@
             (op-mem-layer/init-empty-ops-store! repo)
             (op-mem-layer/update-graph-uuid! repo graph-uuid)
             (op-mem-layer/update-local-tx! repo 8)
-            (m/? (c.m/<! (op-mem-layer/<sync-to-idb-layer! repo)))
+            (m/? (op-mem-layer/new-task--sync-to-idb repo))
             nil)
           (throw (ex-info "upload-graph failed" {:upload-resp upload-resp})))))))
 
@@ -231,6 +231,6 @@
           (op-mem-layer/init-empty-ops-store! repo)
           (m/? (new-task--transact-remote-all-blocks all-blocks repo graph-uuid))
           (op-mem-layer/update-graph-uuid! repo graph-uuid)
-          (m/? (c.m/<! (op-mem-layer/<sync-to-idb-layer! repo)))
+          (m/? (op-mem-layer/new-task--sync-to-idb repo))
           (m/? (c.m/await-promise (.storeMetadata worker-obj repo (pr-str {:graph/uuid graph-uuid}))))
           (worker-state/set-rtc-downloading-graph! false))))))

src/main/frontend/worker/rtc/op_idb_layer.cljs

@@ -1,9 +1,9 @@
 (ns frontend.worker.rtc.op-idb-layer
   "Fns to read/write client-ops from/into indexeddb."
   (:require ["/frontend/idbkv" :as idb-keyval]
-            [cljs.core.async.interop :refer [p->c]]
             [promesa.core :as p]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [logseq.db.sqlite.util :as sqlite-util]
+            [logseq.db :as ldb]))
 
 (def stores (atom {}))
 
@@ -17,36 +17,17 @@
       (do (swap! stores assoc repo (idb-keyval/newStore (str "rtc-ops-" repo) "ops"))
           (@stores repo)))))
 
+(defn <reset2!
+  [repo v]
+  (p/do!
+   (when-let [store (ensure-store repo)]
+     (let [v (ldb/write-transit-str v)]
+       (idb-keyval/set "v" v store)))))
 
-(defn- ops=>idb-items
-  [ops]
-  (keep
-   (fn [op]
-     (when-let [key (:epoch (second op))]
-       {:key key :value op}))
-   ops))
-
-(defn <reset!
-  [repo ops graph-uuid local-tx]
-  (p->c
-   ;; ensure return a promise
-   (p/do!
-    (when-let [store (ensure-store repo)]
-      (let [idb-items (ops=>idb-items ops)]
-        (p/do!
-         (idb-keyval/clear store)
-         (idb-keyval/setBatch (clj->js idb-items) store)
-         (when graph-uuid
-           (idb-keyval/set "graph-uuid" graph-uuid store))
-         (when local-tx
-           (idb-keyval/set "local-tx" local-tx store))))))))
-
-
-(defn <read
+(defn <read2
   [repo]
-  ;; ensure return a promise
   (p/do!
    (when-let [store (ensure-store repo)]
-     (p/let [idb-keys (idb-keyval/keys store)]
-       (-> (p/all (mapv (fn [k] (p/chain (idb-keyval/get k store) (partial vector k))) idb-keys))
-           (p/then (fn [items] (mapv #(js->clj % :keywordize-keys true) items))))))))
+     (p/let [v (idb-keyval/get "v" store)]
+       (when v
+         (ldb/read-transit-str v))))))

src/main/frontend/worker/rtc/op_mem_layer.cljs

@@ -1,89 +1,68 @@
 (ns frontend.worker.rtc.op-mem-layer
   "Store client-ops in memory.
   And sync these ops to indexedDb automatically."
-  (:require [clojure.core.async :as async :refer [<! go-loop timeout]]
-            [clojure.set :as set]
+  (:require [clojure.set :as set]
+            [frontend.common.missionary-util :as c.m]
+            [frontend.worker.rtc.const :as rtc-const]
             [frontend.worker.rtc.op-idb-layer :as op-idb-layer]
             [frontend.worker.state :as worker-state]
-            [malli.core :as m]
+            [logseq.db :as ldb]
+            [logseq.db.sqlite.util :as sqlite-util]
+            [malli.core :as ma]
             [malli.transform :as mt]
-            [promesa.core :as p]
-            [logseq.db.sqlite.util :as sqlite-util]))
+            [missionary.core :as m]
+            [promesa.core :as p]))
 
 (def op-schema
   [:multi {:dispatch first}
-   ["move"
+   [:move
     [:catn
-     [:op :string]
+     [:op :keyword]
+     [:t :int]
      [:value [:map
-              [:block-uuid :uuid]
-              [:epoch :int]]]]]
-   ["remove"
+              [:block-uuid :uuid]]]]]
+   [:remove
     [:catn
-     [:op :string]
+     [:op :keyword]
+     [:t :int]
      [:value [:map
-              [:block-uuid :uuid]
-              [:epoch :int]]]]]
-   ["update"
+              [:block-uuid :uuid]]]]]
+   [:update-page
     [:catn
-     [:op :string]
+     [:op :keyword]
+     [:t :int]
      [:value [:map
-              [:block-uuid :uuid]
-              [:updated-attrs {:optional true}
-               [:map {:closed true}
-                [:schema {:optional true} :nil]
-                [:content {:optional true} :nil]
-                [:link {:optional true} :nil]
-                [:journal-day {:optional true} :nil]
-                [:alias {:optional true} [:map
-                                          [:add {:optional true} [:set :uuid]]
-                                          [:retract {:optional true} [:set :uuid]]]]
-                [:type {:optional true} [:map
-                                         [:add {:optional true} [:set :string]]
-                                         [:retract {:optional true} [:set :string]]]]
-                [:tags {:optional true} [:map
-                                         [:add {:optional true} [:set :uuid]]
-                                         [:retract {:optional true} [:set :uuid]]]]
-                [:properties {:optional true} [:map
-                                               [:add {:optional true} [:set :uuid]]
-                                               [:retract {:optional true} [:set :uuid]]]]]]
-              [:epoch :int]]]]]
-   ["update-page"
+              [:block-uuid :uuid]]]]]
+   [:remove-page
     [:catn
-     [:op :string]
+     [:op :keyword]
+     [:t :int]
      [:value [:map
-              [:block-uuid :uuid]
-              [:epoch :int]]]]]
-   ["remove-page"
+              [:block-uuid :uuid]]]]]
+   [:update
     [:catn
-     [:op :string]
+     [:op :keyword]
+     [:t :int]
      [:value [:map
               [:block-uuid :uuid]
-              [:epoch :int]]]]]
-   ["update-asset"
+              [:av-coll [:sequential rtc-const/av-schema]]]]]]
+
+   [:update-asset
     [:catn
-     [:op :string]
+     [:op :keyword]
+     [:t :int]
      [:value [:map
-              [:asset-uuid :uuid]
-              [:epoch :int]]]]]
-   ["remove-asset"
+              [:asset-uuid :uuid]]]]]
+   [:remove-asset
     [:catn
-     [:op :string]
+     [:op :keyword]
+     [:t :int]
      [:value [:map
-              [:asset-uuid :uuid]
-              [:epoch :int]]]]]])
+              [:asset-uuid :uuid]]]]]])
 
 (def ops-schema [:sequential op-schema])
 
-(def ops-from-store-schema [:sequential [:catn
-                                         [:key :int]
-                                         [:op op-schema]]])
-
-(def ops-from-store-coercer (m/coercer ops-from-store-schema mt/json-transformer))
-(def ops-validator (m/validator ops-schema))
-(def ops-coercer (m/coercer ops-schema mt/json-transformer))
-(def ops-encoder (m/encoder ops-schema mt/json-transformer))
-
+(def ops-coercer (ma/coercer ops-schema mt/json-transformer))
 
 (def ops-store-value-schema
   [:map
@@ -93,8 +72,7 @@
                       [:map-of [:enum :move :remove :update :update-page :remove-page] :any]]]
    [:asset-uuid->ops [:map-of :uuid
                       [:map-of [:enum :update-asset :remove-asset] :any]]]
-   [:epoch->block-uuid-sorted-map [:map-of :int :uuid]]
-   [:epoch->asset-uuid-sorted-map [:map-of :int :uuid]]])
+   [:t+block-uuid-sorted-set [:set [:cat :int :uuid]]]])
 
 (def ops-store-schema
   [:map-of :string                      ; repo-name
@@ -102,190 +80,127 @@
     [:current-branch ops-store-value-schema]
     [:old-branch {:optional true} [:maybe ops-store-value-schema]]]])
 
-(def ops-store-schema-coercer (m/coercer ops-store-schema))
-
+(def ops-store-schema-coercer (ma/coercer ops-store-schema))
 
 (defonce *ops-store (atom {} :validator ops-store-schema-coercer))
 
-(defn- merge-add-retract-maps
-  [m1 m2]
-  (let [{add1 :add retract1 :retract} m1
-        {add2 :add retract2 :retract} m2
-        add (set/union (set/difference add1 retract2) add2)
-        retract (set/union (set/difference retract1 add2) retract2)]
-    (when (or (seq add) (seq retract))
-      (cond-> {}
-        (seq add) (assoc :add add)
-        (seq retract) (assoc :retract retract)))))
-
 (defn- merge-update-ops
-  "op2-epoch > op1-epoch"
   [update-op1 update-op2]
-  {:pre [(= "update" (first update-op1))
-         (= "update" (first update-op2))
-         (= (:block-uuid (second update-op1))
-            (:block-uuid (second update-op2)))]}
-  (let [epoch1 (:epoch (second update-op1))
-        epoch2 (:epoch (second update-op2))]
-    (if (> epoch1 epoch2)
+  {:pre [(= :update (first update-op1))
+         (= :update (first update-op2))
+         (= (:block-uuid (last update-op1))
+            (:block-uuid (last update-op2)))]}
+  (let [t1 (second update-op1)
+        t2 (second update-op2)]
+    (if (> t1 t2)
       (merge-update-ops update-op2 update-op1)
-      (let [updated-attrs1 (:updated-attrs (second update-op1))
-            updated-attrs2 (:updated-attrs (second update-op2))
-            alias (merge-add-retract-maps (:alias updated-attrs1) (:alias updated-attrs2))
-            type (merge-add-retract-maps (:type updated-attrs1) (:type updated-attrs2))
-            tags (merge-add-retract-maps (:tags updated-attrs1) (:tags updated-attrs2))
-            properties (merge-add-retract-maps (:properties updated-attrs1) (:properties updated-attrs2))
-            updated-attrs
-            (cond-> (merge (select-keys updated-attrs1 [:schema :content :link])
-                           (select-keys updated-attrs2 [:schema :content :link]))
-              alias (assoc :alias alias)
-              type (assoc :type type)
-              tags (assoc :tags tags)
-              properties (assoc :properties properties))]
-
-        ["update" {:block-uuid (:block-uuid (second update-op1))
-                   :updated-attrs updated-attrs
-                   :epoch epoch2}]))))
-
-(defn- block-uuid->min-epoch
+      (let [{av-coll1 :av-coll block-uuid :block-uuid} (last update-op1)
+            av-coll2 (:av-coll (last update-op2))]
+        [:update t2
+         {:block-uuid block-uuid
+          :av-coll (concat av-coll1 av-coll2)}]))))
+
+(defn- block-uuid->min-t
   [block-uuid->ops block-uuid]
   (some->> (block-uuid->ops block-uuid)
            vals
-           (map (comp :epoch second))
+           (map second)
            seq
            (apply min)))
 
-(defn- asset-uuid->min-epoch
-  [asset-uuid->ops asset-uuid]
-  (block-uuid->min-epoch asset-uuid->ops asset-uuid))
+(defn- update-t+block-uuid-sorted-set
+  [t+block-uuid-sorted-set old-block-uuid->ops block-uuid->ops block-uuid]
+  (let [origin-min-t (block-uuid->min-t old-block-uuid->ops block-uuid)
+        min-t (block-uuid->min-t block-uuid->ops block-uuid)]
+    (cond-> t+block-uuid-sorted-set
+      origin-min-t (disj [origin-min-t block-uuid])
+      true (conj [min-t block-uuid]))))
 
 (defn ^:large-vars/cleanup-todo add-ops-aux
-  [ops block-uuid->ops epoch->block-uuid-sorted-map asset-uuid->ops epoch->asset-uuid-sorted-map]
+  [ops block-uuid->ops t+block-uuid-sorted-set]
   (loop [block-uuid->ops block-uuid->ops
-         epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map
-         asset-uuid->ops asset-uuid->ops
-         epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map
+         t+block-uuid-sorted-set t+block-uuid-sorted-set
          [op & others] ops]
     (if-not op
       {:block-uuid->ops block-uuid->ops
-       :asset-uuid->ops asset-uuid->ops
-       :epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map
-       :epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map}
-      (let [[op-type value] op
-            {:keys [block-uuid asset-uuid epoch]} value
-            exist-ops (some-> block-uuid block-uuid->ops)
-            exist-asset-ops (some-> asset-uuid asset-uuid->ops)]
+       :t+block-uuid-sorted-set t+block-uuid-sorted-set}
+      (let [[op-type t value] op
+            {:keys [block-uuid]} value
+            exist-ops (some-> block-uuid block-uuid->ops)]
         (case op-type
-          "move"
-          (let [already-removed? (some-> (get exist-ops :remove) second :epoch (> epoch))]
+          :move
+          (let [already-removed? (some-> (get exist-ops :remove) second (> t))]
             (if already-removed?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
+              (recur block-uuid->ops t+block-uuid-sorted-set others)
               (let [block-uuid->ops* (-> block-uuid->ops
                                          (assoc-in [block-uuid :move] op)
                                          (update block-uuid dissoc :remove))
-                    origin-min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)
-                    min-epoch (block-uuid->min-epoch block-uuid->ops* block-uuid)]
-                (recur (-> block-uuid->ops
-                           (assoc-in [block-uuid :move] op)
-                           (update block-uuid dissoc :remove))
-                       (-> epoch->block-uuid-sorted-map
-                           (dissoc origin-min-epoch)
-                           (assoc min-epoch block-uuid))
-                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
-          "update"
-          (let [already-removed? (some-> (get exist-ops :remove) second :epoch (> epoch))]
+                    t+block-uuid-sorted-set*
+                    (update-t+block-uuid-sorted-set t+block-uuid-sorted-set
+                                                    block-uuid->ops
+                                                    block-uuid->ops*
+                                                    block-uuid)]
+                (recur block-uuid->ops* t+block-uuid-sorted-set* others))))
+          :update
+          (let [already-removed? (some-> (get exist-ops :remove) second (> t))]
             (if already-removed?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
+              (recur block-uuid->ops t+block-uuid-sorted-set others)
               (let [origin-update-op (get-in block-uuid->ops [block-uuid :update])
                     op* (if origin-update-op (merge-update-ops origin-update-op op) op)
                     block-uuid->ops* (-> block-uuid->ops
                                          (assoc-in [block-uuid :update] op*)
                                          (update block-uuid dissoc :remove))
-                    origin-min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)
-                    min-epoch (block-uuid->min-epoch block-uuid->ops* block-uuid)]
-                (recur block-uuid->ops*
-                       (-> epoch->block-uuid-sorted-map
-                           (dissoc origin-min-epoch)
-                           (assoc min-epoch block-uuid))
-                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
-          "remove"
-          (let [add-after-remove? (some-> (get exist-ops :move) second :epoch (> epoch))]
+                    t+block-uuid-sorted-set*
+                    (update-t+block-uuid-sorted-set t+block-uuid-sorted-set
+                                                    block-uuid->ops
+                                                    block-uuid->ops*
+                                                    block-uuid)]
+                (recur block-uuid->ops* t+block-uuid-sorted-set* others))))
+          :remove
+          (let [add-after-remove? (some-> (get exist-ops :move) second (> t))]
             (if add-after-remove?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
+              (recur block-uuid->ops t+block-uuid-sorted-set others)
               (let [block-uuid->ops* (assoc block-uuid->ops block-uuid {:remove op})
-                    origin-min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)
-                    min-epoch (block-uuid->min-epoch block-uuid->ops* block-uuid)]
-                (recur block-uuid->ops*
-                       (-> epoch->block-uuid-sorted-map
-                           (dissoc origin-min-epoch)
-                           (assoc min-epoch block-uuid))
-                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
-          "update-page"
-          (let [already-removed? (some-> (get exist-ops :remove-page) second :epoch (> epoch))]
+                    t+block-uuid-sorted-set*
+                    (update-t+block-uuid-sorted-set t+block-uuid-sorted-set
+                                                    block-uuid->ops
+                                                    block-uuid->ops*
+                                                    block-uuid)]
+                (recur block-uuid->ops* t+block-uuid-sorted-set* others))))
+          :update-page
+          (let [already-removed? (some-> (get exist-ops :remove-page) second (> t))]
             (if already-removed?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
+              (recur block-uuid->ops t+block-uuid-sorted-set others)
               (let [block-uuid->ops* (-> block-uuid->ops
                                          (assoc-in [block-uuid :update-page] op)
                                          (update block-uuid dissoc :remove-page))
-                    origin-min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)
-                    min-epoch (block-uuid->min-epoch block-uuid->ops* block-uuid)]
-                (recur block-uuid->ops*
-                       (-> epoch->block-uuid-sorted-map
-                           (dissoc origin-min-epoch)
-                           (assoc min-epoch block-uuid))
-                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
-          "remove-page"
-          (let [add-after-remove? (some-> (get exist-ops :update-page) second :epoch (> epoch))]
+                    t+block-uuid-sorted-set*
+                    (update-t+block-uuid-sorted-set t+block-uuid-sorted-set
+                                                    block-uuid->ops
+                                                    block-uuid->ops*
+                                                    block-uuid)]
+                (recur block-uuid->ops* t+block-uuid-sorted-set* others))))
+          :remove-page
+          (let [add-after-remove? (some-> (get exist-ops :update-page) second (> t))]
             (if add-after-remove?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
+              (recur block-uuid->ops t+block-uuid-sorted-set others)
               (let [block-uuid->ops* (assoc block-uuid->ops block-uuid {:remove-page op})
-                    origin-min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)
-                    min-epoch (block-uuid->min-epoch block-uuid->ops* block-uuid)]
-                (recur block-uuid->ops*
-                       (-> epoch->block-uuid-sorted-map
-                           (dissoc origin-min-epoch)
-                           (assoc min-epoch block-uuid))
-                       asset-uuid->ops epoch->asset-uuid-sorted-map others))))
-          "update-asset"
-          (let [already-removed? (some-> (get exist-asset-ops :remove-asset) second :epoch (> epoch))]
-            (if already-removed?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
-              (let [asset-uuid->ops* (assoc asset-uuid->ops asset-uuid {:update-asset op})
-                    origin-min-epoch (asset-uuid->min-epoch asset-uuid->ops asset-uuid)
-                    min-epoch (asset-uuid->min-epoch asset-uuid->ops* asset-uuid)]
-                (recur block-uuid->ops epoch->block-uuid-sorted-map
-                       asset-uuid->ops*
-                       (-> epoch->asset-uuid-sorted-map
-                           (dissoc origin-min-epoch)
-                           (assoc min-epoch asset-uuid))
-                       others))))
-          "remove-asset"
-          (let [add-after-remove? (some-> (get exist-asset-ops :update-asset) second :epoch (> epoch))]
-            (if add-after-remove?
-              (recur block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map others)
-              (let [asset-uuid->ops* (assoc asset-uuid->ops asset-uuid {:remove-asset op})
-                    origin-min-epoch (asset-uuid->min-epoch asset-uuid->ops asset-uuid)
-                    min-epoch (asset-uuid->min-epoch asset-uuid->ops* asset-uuid)]
-                (recur block-uuid->ops epoch->block-uuid-sorted-map
-                       asset-uuid->ops*
-                       (-> epoch->asset-uuid-sorted-map
-                           (dissoc origin-min-epoch)
-                           (assoc min-epoch asset-uuid))
-                       others))))
-          )))))
-
-
-(def empty-ops-store-value {:current-branch {:block-uuid->ops {}
-                                             :epoch->block-uuid-sorted-map (sorted-map-by <)
-                                             :asset-uuid->ops {}
-                                             :epoch->asset-uuid-sorted-map (sorted-map-by <)}})
+                    t+block-uuid-sorted-set*
+                    (update-t+block-uuid-sorted-set t+block-uuid-sorted-set
+                                                    block-uuid->ops
+                                                    block-uuid->ops*
+                                                    block-uuid)]
+                (recur block-uuid->ops* t+block-uuid-sorted-set* others)))))))))
+
+(def ^:private sorted-set-by-t (sorted-set-by (fn [[t1 x] [t2 y]]
+                                                (let [r (compare t1 t2)]
+                                                  (if (not= r 0)
+                                                    r
+                                                    (compare x y))))))
+
+(def ^:private empty-ops-store-value {:current-branch {:block-uuid->ops {}
+                                                       :asset-uuid->ops {}
+                                                       :t+block-uuid-sorted-set sorted-set-by-t}})
 
 (defn init-empty-ops-store!
   [repo]
@@ -300,62 +215,26 @@
   (assert (contains? (@*ops-store repo) :current-branch) (@*ops-store repo))
   (let [ops (ops-coercer ops)
         {{old-branch-block-uuid->ops :block-uuid->ops
-          old-epoch->block-uuid-sorted-map :epoch->block-uuid-sorted-map
-          old-branch-asset-uuid->ops :asset-uuid->ops
-          old-epoch->asset-uuid-sorted-map :epoch->asset-uuid-sorted-map
+          old-t+block-uuid-sorted-set :t+block-uuid-sorted-set
           :as old-branch} :old-branch
-         {:keys [block-uuid->ops epoch->block-uuid-sorted-map
-                 asset-uuid->ops epoch->asset-uuid-sorted-map]} :current-branch}
+         {:keys [block-uuid->ops t+block-uuid-sorted-set]} :current-branch}
         (get @*ops-store repo)
-        {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-        (add-ops-aux ops block-uuid->ops epoch->block-uuid-sorted-map
-                                    asset-uuid->ops epoch->asset-uuid-sorted-map)
-        {old-branch-block-uuid->ops :block-uuid->ops old-epoch->block-uuid-sorted-map :epoch->block-uuid-sorted-map}
+        {:keys [block-uuid->ops t+block-uuid-sorted-set]}
+        (add-ops-aux ops block-uuid->ops t+block-uuid-sorted-set)
+        {old-branch-block-uuid->ops :block-uuid->ops old-t+block-uuid-sorted-set :t+block-uuid-sorted-set}
         (when old-branch
-          (add-ops-aux ops old-branch-block-uuid->ops old-epoch->block-uuid-sorted-map
-                                      old-branch-asset-uuid->ops old-epoch->asset-uuid-sorted-map))]
+          (add-ops-aux ops old-branch-block-uuid->ops old-t+block-uuid-sorted-set))]
     (swap! *ops-store update repo
            (fn [{:keys [current-branch old-branch]}]
              (cond-> {:current-branch
                       (assoc current-branch
                              :block-uuid->ops block-uuid->ops
-                             :epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map)}
+                             :t+block-uuid-sorted-set t+block-uuid-sorted-set)}
                old-branch
                (assoc :old-branch
                       (assoc old-branch
                              :block-uuid->ops old-branch-block-uuid->ops
-                             :epoch->block-uuid-sorted-map old-epoch->block-uuid-sorted-map)))))))
-
-(defn add-asset-ops!
-  [repo ops]
-  (assert (contains? (@*ops-store repo) :current-branch) (@*ops-store repo))
-  (let [ops (ops-coercer ops)
-        {{old-branch-block-uuid->ops :block-uuid->ops
-          old-epoch->block-uuid-sorted-map :epoch->block-uuid-sorted-map
-          old-branch-asset-uuid->ops :asset-uuid->ops
-          old-epoch->asset-uuid-sorted-map :epoch->asset-uuid-sorted-map
-          :as old-branch} :old-branch
-         {:keys [block-uuid->ops epoch->block-uuid-sorted-map
-                 asset-uuid->ops epoch->asset-uuid-sorted-map]} :current-branch}
-        (get @*ops-store repo)
-        {:keys [asset-uuid->ops epoch->asset-uuid-sorted-map]}
-        (add-ops-aux ops block-uuid->ops epoch->block-uuid-sorted-map
-                     asset-uuid->ops epoch->asset-uuid-sorted-map)
-        {old-branch-asset-uuid->ops :asset-uuid->ops old-epoch->asset-uuid-sorted-map :epoch->asset-uuid-sorted-map}
-        (when old-branch
-          (add-ops-aux ops old-branch-block-uuid->ops old-epoch->block-uuid-sorted-map
-                       old-branch-asset-uuid->ops old-epoch->asset-uuid-sorted-map))]
-    (swap! *ops-store update repo
-           (fn [{:keys [current-branch old-branch]}]
-             (cond-> {:current-branch
-                      (assoc current-branch
-                             :asset-uuid->ops asset-uuid->ops
-                             :epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map)}
-               old-branch
-               (assoc :old-branch
-                      (assoc old-branch
-                             :asset-uuid->ops old-branch-asset-uuid->ops
-                             :epoch->asset-uuid-sorted-map old-epoch->asset-uuid-sorted-map)))))))
+                             :t+block-uuid-sorted-set old-t+block-uuid-sorted-set)))))))
 
 (defn update-local-tx!
   [repo t]
@@ -390,16 +269,22 @@
   [repo]
   (swap! *ops-store update repo dissoc :old-branch))
 
-
-(defn get-min-epoch-block-ops
+(defn get-min-t-block-ops
   [repo]
   (let [repo-ops-store (get @*ops-store repo)
-        {:keys [epoch->block-uuid-sorted-map block-uuid->ops]} (:current-branch repo-ops-store)]
+        {:keys [t+block-uuid-sorted-set block-uuid->ops]} (:current-branch repo-ops-store)]
     (assert (contains? repo-ops-store :current-branch) repo)
-    (when-let [[_epoch block-uuid] (first epoch->block-uuid-sorted-map)]
-      (assert (contains? block-uuid->ops block-uuid))
-      {:block-uuid block-uuid
-       :ops (block-uuid->ops block-uuid)})))
+    (when-let [[t block-uuid] (first t+block-uuid-sorted-set)]
+      (if (contains? block-uuid->ops block-uuid)
+        {:block-uuid block-uuid
+         :ops (block-uuid->ops block-uuid)}
+
+        (throw (ex-info "unavailable" {:t t :block-uuid block-uuid :block-uuid->ops block-uuid->ops}))
+        ;; if not found, remove item in :t+block-uuid-sorted-set and retry
+        ;; (do (swap! *ops-store update-in [repo :current-branch] assoc
+        ;;            :t+block-uuid-sorted-set (disj t+block-uuid-sorted-set [t block-uuid]))
+        ;;     (get-min-t-block-ops repo))
+        ))))
 
 (defn get-block-ops
   [repo block-uuid]
@@ -416,25 +301,6 @@
            vals
            (mapcat vals)))
 
-(comment
-  (defn get-min-epoch-asset-ops
-    [repo]
-    (let [repo-ops-store (get @*ops-store repo)
-          {:keys [epoch->asset-uuid-sorted-map asset-uuid->ops]} (:current-branch repo-ops-store)]
-      (assert (contains? repo-ops-store :current-branch) repo)
-      (when-let [[_epoch asset-uuid] (first epoch->asset-uuid-sorted-map)]
-        (assert (contains? asset-uuid->ops asset-uuid))
-        {:asset-uuid asset-uuid
-         :ops (asset-uuid->ops asset-uuid)}))))
-
-(comment
-  (defn get-asset-ops
-    [repo asset-uuid]
-    (let [repo-ops-store (get @*ops-store repo)
-          {:keys [asset-uuid->ops]} (:current-branch repo-ops-store)]
-      (assert (contains? repo-ops-store :current-branch) repo)
-      (asset-uuid->ops asset-uuid))))
-
 (defn get-local-tx
   [repo]
   (some-> (get @*ops-store repo)
@@ -469,78 +335,53 @@
            set
            (set/intersection (set block-uuid-coll))))
 
-
 (defn remove-block-ops!
   [repo block-uuid]
   {:pre [(uuid? block-uuid)]}
   (let [repo-ops-store (get @*ops-store repo)
-        {:keys [epoch->block-uuid-sorted-map block-uuid->ops]} (:current-branch repo-ops-store)]
+        {:keys [t+block-uuid-sorted-set block-uuid->ops]} (:current-branch repo-ops-store)]
     (assert (contains? repo-ops-store :current-branch) repo)
-    (let [min-epoch (block-uuid->min-epoch block-uuid->ops block-uuid)]
+    (let [min-t (block-uuid->min-t block-uuid->ops block-uuid)]
       (swap! *ops-store update-in [repo :current-branch] assoc
              :block-uuid->ops (dissoc block-uuid->ops block-uuid)
-             :epoch->block-uuid-sorted-map (dissoc epoch->block-uuid-sorted-map min-epoch)))))
+             :t+block-uuid-sorted-set (disj t+block-uuid-sorted-set [min-t block-uuid])))))
 
-(comment
-  (defn remove-asset-ops!
-    [repo asset-uuid]
-    {:pre [(uuid? asset-uuid)]}
-    (let [repo-ops-store (get @*ops-store repo)
-          {:keys [epoch->asset-uuid-sorted-map asset-uuid->ops]} (:current-branch repo-ops-store)]
-      (assert (contains? repo-ops-store :current-branch) repo)
-      (let [min-epoch (asset-uuid->min-epoch asset-uuid->ops asset-uuid)]
-        (swap! *ops-store update-in [repo :current-branch] assoc
-               :asset-uuid->ops (dissoc asset-uuid->ops asset-uuid)
-               :epoch->asset-uuid-sorted-map (dissoc epoch->asset-uuid-sorted-map min-epoch))))))
-
-
-(defn <init-load-from-indexeddb!
+
+(defn <init-load-from-indexeddb2!
   [repo]
-  (p/let [all-data (op-idb-layer/<read repo)
-          all-data-m (into {} all-data)
-          local-tx (get all-data-m "local-tx")]
-    (when local-tx
-      (let [graph-uuid (get all-data-m "graph-uuid")
-            ops (->> all-data
-                     (filter (comp number? first))
-                     (sort-by first <)
-                     ops-from-store-coercer
-                     (map second))
-            {:keys [block-uuid->ops epoch->block-uuid-sorted-map asset-uuid->ops epoch->asset-uuid-sorted-map]}
-            (add-ops-aux ops {} (sorted-map-by <) {} (sorted-map-by <))
-            r (cond-> {:block-uuid->ops block-uuid->ops
-                       :epoch->block-uuid-sorted-map epoch->block-uuid-sorted-map
-                       :asset-uuid->ops asset-uuid->ops
-                       :epoch->asset-uuid-sorted-map epoch->asset-uuid-sorted-map}
-                graph-uuid (assoc :graph-uuid graph-uuid)
-                local-tx (assoc :local-tx local-tx))]
-        (assert (ops-validator ops) ops)
-        (swap! *ops-store update repo #(-> %
-                                           (assoc :current-branch r)
-                                           (dissoc :old-branch)))
+  (p/let [v (op-idb-layer/<read2 repo)]
+    (when v
+      (let [v (assoc v
+                     :t+block-uuid-sorted-set
+                     (apply conj sorted-set-by-t (:t+block-uuid-sorted-set v)))]
+        (swap! *ops-store assoc repo {:current-branch v})
         (prn ::<init-load-from-indexeddb! repo)))))
 
-(defn <sync-to-idb-layer!
+(defn new-task--sync-to-idb
   [repo]
-  (let [repo-ops-store (get @*ops-store repo)
-        {:keys [block-uuid->ops local-tx graph-uuid]} (:current-branch repo-ops-store)
-        ops (mapcat vals (vals block-uuid->ops))
-        ops* (ops-encoder ops)]
-    (op-idb-layer/<reset! repo ops* graph-uuid local-tx)))
+  (m/sp
+    (when-let [v (:current-branch (@*ops-store repo))]
+      (m/? (c.m/await-promise (op-idb-layer/<reset2! repo v))))))
 
-(defn run-sync-loop
+(defn- new-task--sync-to-idb-loop
   []
-  (go-loop []
-    (<! (timeout 3000))
-    (when-let [repo (worker-state/get-current-repo)]
-      (when (and (sqlite-util/db-based-graph? repo)
-                 (contains? (@*ops-store repo) :current-branch))
-        (<! (<sync-to-idb-layer! repo))))
-    (recur)))
+  (m/sp
+    (let [*v-hash (atom nil)]
+      (loop []
+        (m/? (m/sleep 3000))
+        (let [repo (worker-state/get-current-repo)
+              conn (worker-state/get-datascript-conn repo)]
+          (when (and repo conn
+                     (ldb/db-based-graph? @conn))
+            (when-let [v (:current-branch (@*ops-store repo))]
+              (let [v-hash (hash v)]
+                (when (not= v-hash @*v-hash)
+                  (m/? (c.m/await-promise (op-idb-layer/<reset2! repo v)))
+                  (reset! *v-hash v-hash))))))
+        (recur)))))
 
 #_:clj-kondo/ignore
-(defonce _sync-loop (run-sync-loop))
-
+(defonce _sync-loop-canceler (c.m/run-task (new-task--sync-to-idb-loop) ::sync-to-idb-loop))
 
 (defn rtc-db-graph?
   "Is db-graph & RTC enabled"

src/test/frontend/worker/rtc/db_listener_test.cljs

@@ -5,17 +5,15 @@
             [frontend.worker.rtc.db-listener :as subject]
             [logseq.db.frontend.schema :as db-schema]))
 
-
 (def empty-db (d/empty-db db-schema/schema-for-db-based-graph))
 
-
 (defn- tx-data=>id->attr->datom
   [tx-data]
   (let [datom-vec-coll (map vec tx-data)
         id->same-entity-datoms (group-by first datom-vec-coll)]
     (update-vals id->same-entity-datoms #'worker-db-listener/entity-datoms=>attr->datom)))
 
-(deftest entity-datoms=>ops-test
+(deftest ^:fix-me entity-datoms=>ops-test
   (testing "remove whiteboard page-block"
     (let [conn (d/conn-from-db empty-db)
           block-uuid (random-uuid)
@@ -27,7 +25,7 @@
           remove-whiteboard-page-block
           (d/transact! conn [[:db/retractEntity [:block/uuid block-uuid]]])]
       (is (= [["remove-page" {:block-uuid (str block-uuid)}]]
-             (#'subject/entity-datoms=>ops (:db-before remove-whiteboard-page-block)
-                                           (:db-after remove-whiteboard-page-block)
-                                           (tx-data=>id->attr->datom (:tx-data remove-whiteboard-page-block))
-                                           (map vec (:tx-data remove-whiteboard-page-block))))))))
+             (#'subject/entity-datoms=>ops2 (:db-before remove-whiteboard-page-block)
+                                            (:db-after remove-whiteboard-page-block)
+                                            (tx-data=>id->attr->datom (:tx-data remove-whiteboard-page-block))
+                                            (map vec (:tx-data remove-whiteboard-page-block))))))))

src/test/frontend/worker/rtc/idb_keyval_mock.clj

@@ -1,16 +0,0 @@
-(ns frontend.worker.rtc.idb-keyval-mock
-  (:require [frontend.test.helper :as test-helper]))
-
-
-
-(defmacro with-reset-idb-keyval-mock
-  [reset & body]
-  `(test-helper/with-reset ~reset
-     [idb-keyval/set      frontend.worker.rtc.idb-keyval-mock/set
-      idb-keyval/setBatch frontend.worker.rtc.idb-keyval-mock/set-batch
-      idb-keyval/get      frontend.worker.rtc.idb-keyval-mock/get
-      idb-keyval/del      frontend.worker.rtc.idb-keyval-mock/del
-      idb-keyval/keys     frontend.worker.rtc.idb-keyval-mock/keys
-      idb-keyval/clear    frontend.worker.rtc.idb-keyval-mock/clear
-      idb-keyval/newStore frontend.worker.rtc.idb-keyval-mock/new-store]
-     ~@body))

src/test/frontend/worker/rtc/idb_keyval_mock.cljs

@@ -1,35 +0,0 @@
-(ns frontend.worker.rtc.idb-keyval-mock
-  "Mock fns for frontend/idbkv.js"
-  (:refer-clojure :exclude [get set keys])
-  (:require [promesa.core :as p]))
-
-(defrecord Store [db-name store-name *kvs])
-
-(defn new-store [db-name store-name & _args]
-  (Store. db-name store-name (atom {})))
-
-(defn get
-  [key store]
-  (p/do! (clojure.core/get @(:*kvs store) key)))
-
-(defn set
-  [key val store]
-  (p/do! (swap! (:*kvs store) assoc key val)))
-
-(defn set-batch
-  [items store]
-  (p/do!
-   (let [kvs (mapcat (fn [x] [(.-key x) (.-value x)]) items)]
-     (swap! (:*kvs store) (partial apply assoc) kvs))))
-
-(defn del
-  [key store]
-  (p/do! (swap! (:*kvs store) dissoc key)))
-
-(defn keys
-  [store]
-  (p/do! (clojure.core/keys @(:*kvs store))))
-
-(defn clear
-  [store]
-  (p/do! (reset! (:*kvs store) {})))

src/test/frontend/worker/rtc/op_mem_layer_test.cljs

@@ -1,218 +1,88 @@
 (ns frontend.worker.rtc.op-mem-layer-test
   (:require [cljs.test :as t :refer [deftest is testing]]
-            [clojure.core.async :as async :refer [<! go]]
-            [cljs.core.async.interop :refer [p->c]]
-            [frontend.worker.rtc.idb-keyval-mock :include-macros true :as idb-keyval-mock]
-            [frontend.worker.rtc.op-idb-layer :as op-idb-layer]
             [frontend.worker.rtc.op-mem-layer :as op-layer]
-            #_:clj-kondo/ignore ["/frontend/idbkv" :as idb-keyval]
-            [frontend.config :as config]))
+            [frontend.worker.rtc.fixture :as fixture]
+            [frontend.test.helper :as test-helper]))
 
-(defn- make-db-graph-repo-name
-  [s]
-  (str config/db-version-prefix s))
+(t/use-fixtures :each fixture/clear-op-mem-stores-fixture)
 
-(deftest add-ops-to-block-uuid->ops-test
-  (testing "case1"
-    (let [ops [["move" {:block-uuid "f4abd682-fb9e-4f1a-84bf-5fe11fe7844b" :epoch 1}]
-               ["move" {:block-uuid "8e6d8355-ded7-4500-afaa-6f721f3b0dc6" :epoch 2}]]
-          {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-          (op-layer/add-ops-aux (op-layer/ops-coercer ops) {} (sorted-map-by <) {} (sorted-map-by <))]
-      (is (= [{#uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"
-               {:move ["move" {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b", :epoch 1}]},
-               #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"
-               {:move ["move" {:block-uuid #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6", :epoch 2}]}}
-              {1 #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b", 2 #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"}]
-             [block-uuid->ops epoch->block-uuid-sorted-map]))))
+(def ^:private repo test-helper/test-db-name-db-version)
 
-  (testing "case2"
-    (let [ops [["move" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 1}]
-               ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 2
-                          :updated-attrs {:content nil}}]
-               ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 4
-                          :updated-attrs {:type {:add #{"type2" "type3"} :retract #{"type1"}}}}]
-               ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 3
-                          :updated-attrs {:type {:add #{"type1"}}}}]]
-          {:keys [block-uuid->ops epoch->block-uuid-sorted-map]}
-          (op-layer/add-ops-aux (op-layer/ops-coercer ops) {} (sorted-map-by <) {} (sorted-map-by <))]
-      (is (= [{#uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"
-               {:move
-                ["move" {:block-uuid #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02", :epoch 1}],
+(deftest add-ops!-test
+  (let [block-uuid #uuid "663dd373-41b0-4c59-aa6f-f349eea82609"]
+    (testing "move+update on same block"
+      (let [ops [[:move
+                  1
+                  {:block-uuid block-uuid}]
+                 [:update
+                  1
+                  {:block-uuid block-uuid,
+                   :av-coll
+                   [[:block/updated-at "[\"~#'\",1715327859860]" 1 true]
+                    [:block/created-at "[\"~#'\",1715327859860]" 1 true]
+                    [:block/content "[\"~#'\",\"\"]" 1 true]]}]]]
+        (op-layer/add-ops! repo ops)
+        (is (= {:move
+                [:move
+                 1
+                 {:block-uuid block-uuid}],
                 :update
-                ["update" {:block-uuid #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02",
-                           :updated-attrs
-                           {:content nil, :type {:add #{"type2" "type3"}, :retract #{"type1"}}},
-                           :epoch 4}]}}
-              {1 #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"}]
-             [block-uuid->ops epoch->block-uuid-sorted-map]))))
-  (testing "case3: :link"
-    (let [ops [["move" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 1}]
-               ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 2
-                          :updated-attrs {:content nil}}]
-               ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 4
-                          :updated-attrs {:content nil :link nil}}]]
-          {:keys [block-uuid->ops]}
-          (op-layer/add-ops-aux (op-layer/ops-coercer ops) {} (sorted-map-by <) {} (sorted-map-by <))]
-      (is (= ["update"
-              {:block-uuid #uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02"
-               :updated-attrs {:content nil :link nil}
-               :epoch 4}]
-             (:update (block-uuid->ops #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"))))))
-  (testing "case4: update-page then remove-page"
-    (let [ops1 [["update-page" {:block-uuid #uuid "65564abe-1e79-4ae8-af60-215826cefea9" :epoch 1}]]
-          ops2 [["remove-page" {:block-uuid #uuid "65564abe-1e79-4ae8-af60-215826cefea9" :epoch 2}]]
-          {:keys [block-uuid->ops epoch->block-uuid-sorted-map asset-uuid->ops epoch->asset-uuid-sorted-map]}
-          (op-layer/add-ops-aux (op-layer/ops-coercer ops1) {} (sorted-map-by <) {} (sorted-map-by <))
-          {block-uuid->ops2 :block-uuid->ops}
-          (op-layer/add-ops-aux (op-layer/ops-coercer ops2)
-                                               block-uuid->ops epoch->block-uuid-sorted-map
-                                               asset-uuid->ops epoch->asset-uuid-sorted-map)]
-      (is (= {#uuid "65564abe-1e79-4ae8-af60-215826cefea9"
-              {:remove-page ["remove-page" {:block-uuid #uuid "65564abe-1e79-4ae8-af60-215826cefea9", :epoch 2}]}}
-             block-uuid->ops2)))))
+                [:update
+                 1
+                 {:block-uuid block-uuid,
+                  :av-coll
+                  [[:block/updated-at "[\"~#'\",1715327859860]" 1 true]
+                   [:block/created-at "[\"~#'\",1715327859860]" 1 true]
+                   [:block/content "[\"~#'\",\"\"]" 1 true]]}]}
+               (op-layer/get-block-ops repo block-uuid)))))
 
-(deftest add-ops-to-asset-uuid->ops-test
-  (let [[uuid1 uuid2] (repeatedly random-uuid)
-        ops1 [["update-asset" {:asset-uuid uuid1 :epoch 1}]
-              ["update-asset" {:asset-uuid uuid2 :epoch 2}]]
-        {:keys [asset-uuid->ops]}
-        (op-layer/add-ops-aux (op-layer/ops-coercer ops1) {} (sorted-map-by <) {} (sorted-map-by <))]
-    (is (= {uuid1 {:update-asset ["update-asset" {:asset-uuid uuid1 :epoch 1}]}
-            uuid2 {:update-asset ["update-asset" {:asset-uuid uuid2 :epoch 2}]}}
-           asset-uuid->ops))))
+    (testing "more updates on this block"
+      (let [ops [[:update
+                  2
+                  {:block-uuid block-uuid,
+                   :av-coll
+                   [[:block/updated-at "[\"~#'\",1715327859860]" 2 false]
+                    [:block/updated-at "[\"~#'\",1715329245395]" 2 true]
+                    [:block/content "[\"~#'\",\"\"]" 2 false]
+                    [:block/content "[\"~#'\",\"iii\"]" 2 true]]}]
+                 [:update
+                  3
+                  {:block-uuid block-uuid,
+                   :av-coll
+                   [[:block/tags #uuid "663dd8e0-8840-4411-ab6f-2632ac36bf11" 3 true]]}]]]
+        (op-layer/add-ops! repo ops)
+        (is (=
+             {:move
+              [:move
+               1
+               {:block-uuid block-uuid}],
+              :update
+              [:update
+               3
+               {:block-uuid block-uuid,
+                :av-coll
+                [[:block/updated-at "[\"~#'\",1715327859860]" 1 true]
+                 [:block/created-at "[\"~#'\",1715327859860]" 1 true]
+                 [:block/content "[\"~#'\",\"\"]" 1 true]
+                 [:block/updated-at "[\"~#'\",1715327859860]" 2 false]
+                 [:block/updated-at "[\"~#'\",1715329245395]" 2 true]
+                 [:block/content "[\"~#'\",\"\"]" 2 false]
+                 [:block/content "[\"~#'\",\"iii\"]" 2 true]
+                 [:block/tags #uuid "663dd8e0-8840-4411-ab6f-2632ac36bf11" 3 true]]}]}
+             (op-layer/get-block-ops repo block-uuid)))
+        (is (= [1 block-uuid]
+               (first (:t+block-uuid-sorted-set (:current-branch (get @op-layer/*ops-store repo))))))))
 
+    (testing "insert some other blocks"
+      (let [block-uuids (repeatedly 3 random-uuid)
+            ops (map-indexed (fn [idx block-uuid]
+                               [:move (+ idx 4) {:block-uuid block-uuid}]) block-uuids)]
+        (op-layer/add-ops! repo ops)
+        (is (= block-uuid (:block-uuid (op-layer/get-min-t-block-ops repo))))))
 
-(deftest process-test
-  (let [repo (make-db-graph-repo-name "process-test")
-        ops1 [["move" {:block-uuid "f4abd682-fb9e-4f1a-84bf-5fe11fe7844b" :epoch 1}]
-              ["move" {:block-uuid "8e6d8355-ded7-4500-afaa-6f721f3b0dc6" :epoch 2}]]
-        ops2 [["update" {:block-uuid "f4abd682-fb9e-4f1a-84bf-5fe11fe7844b" :epoch 3
-                         :updated-attrs {:content nil}}]]
-        ops3 [["update" {:block-uuid "f4abd682-fb9e-4f1a-84bf-5fe11fe7844b" :epoch 4
-                         :updated-attrs {:type {:add #{"type1"}}}}]]]
-    (op-layer/init-empty-ops-store! repo)
-    (op-layer/new-branch! repo)
-    (let [{:keys [current-branch old-branch]} (@@#'op-layer/*ops-store repo)]
-      (is (= current-branch old-branch)))
-    (op-layer/add-ops! repo ops1)
-    (op-layer/add-ops! repo ops2)
-    (op-layer/update-local-tx! repo 10)
-    (op-layer/update-graph-uuid! repo "b82c6c92-2d0f-4214-9411-3e9bdc2cefa6")
-    (let [{:keys [current-branch old-branch]} (@@#'op-layer/*ops-store repo)]
-      (is (not= (:local-tx current-branch) (:local-tx old-branch)))
-      (is (= {#uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"
-              {:move
-               ["move"
-                {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b", :epoch 1}],
-               :update
-               ["update"
-                {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b",
-                 :epoch 3,
-                 :updated-attrs {:content nil}}]},
-              #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"
-              {:move
-               ["move"
-                {:block-uuid #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6", :epoch 2}]}}
-             (:block-uuid->ops current-branch)))
-      (is (= {1 #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b" 2 #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"}
-             (:epoch->block-uuid-sorted-map current-branch))))
-    (let [min-epoch-block-ops (op-layer/get-min-epoch-block-ops repo)]
-      (is (= {:ops {:move ["move" {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b", :epoch 1}]
-                    :update ["update" {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b", :epoch 3,
-                                       :updated-attrs {:content nil}}]}
-              :block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"}
-             min-epoch-block-ops)))
-    (op-layer/remove-block-ops! repo #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b")
-    (let [{:keys [current-branch]} (@@#'op-layer/*ops-store repo)]
-      (is (= {#uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"
-              {:move ["move" {:block-uuid #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6", :epoch 2}]}}
-             (:block-uuid->ops current-branch)))
-      (is (= {2 #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"}
-             (:epoch->block-uuid-sorted-map current-branch))))
-    (op-layer/add-ops! repo ops3)
-    (let [{:keys [current-branch old-branch]} (@@#'op-layer/*ops-store repo)]
-      (is (= {#uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"
-              {:move
-               ["move"
-                {:block-uuid #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6", :epoch 2}]},
-              #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"
-              {:update
-               ["update"
-                {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b",
-                 :epoch 4,
-                 :updated-attrs {:type {:add #{"type1"}}}}]}}
-             (:block-uuid->ops current-branch)))
-      (is (= {2 #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"
-              4 #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"}
-             (:epoch->block-uuid-sorted-map current-branch)))
-      (is (= {#uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"
-              {:move
-               ["move"
-                {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b", :epoch 1}],
-               :update
-               ["update"
-                {:block-uuid #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b",
-                 :updated-attrs
-                 {:content nil, :type {:add #{"type1"}}},
-                 :epoch 4}]},
-              #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"
-              {:move
-               ["move"
-                {:block-uuid #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6", :epoch 2}]}}
-             (:block-uuid->ops old-branch)))
-      (is (= {1 #uuid"f4abd682-fb9e-4f1a-84bf-5fe11fe7844b"
-              2 #uuid"8e6d8355-ded7-4500-afaa-6f721f3b0dc6"}
-             (:epoch->block-uuid-sorted-map old-branch)))
-      (op-layer/rollback! repo)
-      (let [{current-branch* :current-branch} (@@#'op-layer/*ops-store repo)]
-        (is (= current-branch* old-branch))))
-    (op-layer/remove-ops-store! repo)))
-
-
-
-(deftest load-from&sync-to-idb-test
-  (t/async
-   done
-   (idb-keyval-mock/with-reset-idb-keyval-mock reset
-     (go
-       (let [repo (make-db-graph-repo-name "load-from&sync-to-idb-test")
-             ops [["move" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 1}]
-                  ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 2
-                             :updated-attrs {:content nil}}]
-                  ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 4
-                             :updated-attrs {:type {:add #{"type2" "type3"} :retract #{"type1"}}}}]
-                  ["update" {:block-uuid "f639f13e-ef6f-4ba5-83b4-67527d27cd02" :epoch 3
-                             :updated-attrs {:type {:add #{"type1"}}
-                                             :tags {:add #{#uuid "b0bed412-ad52-4d87-8a08-80ac537e1b61"}}}}]]]
-         (swap! op-idb-layer/stores dissoc repo)
-         (op-layer/init-empty-ops-store! repo)
-         (op-layer/add-ops! repo ops)
-         (op-layer/update-local-tx! repo 1)
-         (let [repo-ops-store1 (@@#'op-layer/*ops-store repo)]
-           (<! (op-layer/<sync-to-idb-layer! repo))
-           (op-layer/remove-ops-store! repo)
-           (<! (p->c (op-layer/<init-load-from-indexeddb! repo)))
-           (let [repo-ops-store2 (@@#'op-layer/*ops-store repo)]
-             (is (= {:current-branch
-                     {:block-uuid->ops
-                      {#uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"
-                       {:move
-                        ["move"
-                         {:epoch 1, :block-uuid #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"}],
-                        :update
-                        ["update"
-                         {:block-uuid #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02",
-                          :updated-attrs
-                          {:content nil,
-                           :type {:add #{"type3" "type2"}, :retract #{"type1"}}
-                           :tags {:add #{#uuid "b0bed412-ad52-4d87-8a08-80ac537e1b61"}}},
-                          :epoch 4}]}},
-                      :epoch->block-uuid-sorted-map
-                      {1 #uuid"f639f13e-ef6f-4ba5-83b4-67527d27cd02"}
-                      :asset-uuid->ops {}
-                      :epoch->asset-uuid-sorted-map {}
-                      :local-tx 1}}
-                    repo-ops-store1))
-             (is (= repo-ops-store1 repo-ops-store2)))))
-       (reset)
-       (done)))))
+    (testing "remove this block"
+      (let [ops [[:remove 999 {:block-uuid block-uuid}]]]
+        (op-layer/add-ops! repo ops)
+        (is (=
+             {:remove [:remove 999 {:block-uuid block-uuid}]}
+             (op-layer/get-block-ops repo block-uuid)))))))