fix: normalize reversed tx-data

deps/db-sync/src/logseq/db_sync/compare.cljs

@@ -35,7 +35,7 @@
   (->> tx-data
        (mapcat
         (fn [[op e a v]]
-          (if (= op :db.fn/retractEntity)
+          (if (= op :db/retractEntity)
             (let [entity (d/entity db-before e)]
               (keep
                (fn [a]

deps/db/src/logseq/db.cljs

@@ -169,7 +169,7 @@
                                        (integer? m)
                                        (empty? m)))))
          delete-blocks-tx (when-not (string? repo-or-conn)
-                            (delete-blocks/update-refs-history-and-macros @repo-or-conn tx-data tx-meta))
+                            (delete-blocks/update-refs-history @repo-or-conn tx-data tx-meta))
          tx-data (concat tx-data delete-blocks-tx)]
 
      ;; Ensure worker can handle the request sequentially (one by one)
@@ -187,7 +187,7 @@
 (defn transact-with-temp-conn!
   "Validate db and store once for a batch transaction, the `temp` conn can still load data from disk,
   however it can't write to the disk."
-  [conn tx-meta batch-tx-fn & {:keys [listen-db]}]
+  [conn tx-meta batch-tx-fn & {:keys [listen-db filter-tx-data]}]
   (let [temp-conn (d/conn-from-db @conn)
         *batch-tx-data (volatile! [])]
     ;; can read from disk, write is disallowed
@@ -200,13 +200,17 @@
                  (when (fn? listen-db)
                    (listen-db tx-report))))
     (batch-tx-fn temp-conn *batch-tx-data)
-    (let [tx-data @*batch-tx-data]
+    (let [tx-data @*batch-tx-data
+          temp-after-db @temp-conn]
       (d/unlisten! temp-conn ::temp-conn-batch-tx)
       (reset! temp-conn nil)
       (vreset! *batch-tx-data nil)
       (when (seq tx-data)
         ;; transact tx-data to `conn` and validate db
-        (transact! conn tx-data tx-meta)))))
+        (let [tx-data' (if (fn? filter-tx-data)
+                         (filter-tx-data temp-after-db tx-data)
+                         tx-data)]
+          (transact! conn tx-data' tx-meta))))))
 
 (def page? common-entity-util/page?)
 (def internal-page? entity-util/internal-page?)

deps/db/src/logseq/db/common/delete_blocks.cljs

@@ -47,9 +47,8 @@
          tx))
      refs)))
 
-(defn update-refs-history-and-macros
-  "When a block is deleted, refs are updated, property history are deleted. For file graphs, macros associated
-  with the block are also deleted"
+(defn update-refs-history
+  "When a block is deleted, refs are updated, property history are deleted."
   [db txs _opts]
   (let [retracted-block-ids (->> (keep (fn [tx]
                                          (when (and (vector? tx)
@@ -63,12 +62,11 @@
             retract-history-tx (mapcat (fn [e]
                                          (map (fn [history] [:db/retractEntity (:db/id history)])
                                               (:logseq.property.history/_block e))) retracted-blocks)
-            macros-tx (when-not (entity-plus/db-based-graph? db)
-                        (mapcat (fn [b]
-                                  ;; Only delete if last reference
-                                  (keep #(when (<= (count (:block/_macros (d/entity db (:db/id %))))
-                                                   1)
-                                           (when (:db/id %) (vector :db.fn/retractEntity (:db/id %))))
-                                        (:block/macros b)))
-                                retracted-blocks))]
-        (concat retracted-tx retract-history-tx macros-tx)))))
+            delete-views (->>
+                          (mapcat
+                           (fn [item]
+                             (let [block (d/entity db (:db/id item))]
+                               (:logseq.property/_view-for block)))
+                           retracted-blocks)
+                          (map (fn [b] [:db/retractEntity (:db/id b)])))]
+        (concat retracted-tx delete-views retract-history-tx)))))

deps/db/src/logseq/db/common/normalize.cljs

@@ -5,7 +5,7 @@
 (defn- remove-retract-entity-ref
   [tx-data]
   (let [retracted (-> (keep (fn [[op value]]
-                              (when (= op :db.fn/retractEntity)
+                              (when (= op :db/retractEntity)
                                 value)) tx-data)
                       set)]
     (if (seq retracted)
@@ -21,7 +21,7 @@
    (map (fn [[op eid a v t]]
           (cond
             (and (= op :db/retract) (= a :block/uuid))
-            [:db.fn/retractEntity eid]
+            [:db/retractEntity eid]
             (and a (some? v))
             [op eid a v t]
             :else
@@ -29,7 +29,7 @@
         normalized-tx-data)
    remove-retract-entity-ref))
 
-(defn- replace-attr-retract-with-retract-entity
+(defn replace-attr-retract-with-retract-entity
   [tx-data]
   (let [e-datoms (->> (group-by first tx-data)
                       (sort-by first))]
@@ -38,7 +38,7 @@
        (if-let [d (some (fn [d]
                           (when (and (= :block/uuid (:a d)) (false? (:added d)))
                             d)) datoms)]  ; retract entity
-         [[:db.fn/retractEntity [:block/uuid (:v d)]]]
+         [[:db/retractEntity [:block/uuid (:v d)]]]
          datoms))
      e-datoms)))
 

deps/graph-parser/src/logseq/graph_parser.cljs

@@ -15,7 +15,7 @@
   (mapcat (fn [{uuid' :block/uuid eid :db/id}]
             (if (and uuid' (contains? retain-uuids uuid'))
               (map (fn [attr] [:db.fn/retractAttribute eid attr]) file-schema/retract-attributes)
-              (when eid [[:db.fn/retractEntity eid]])))
+              (when eid [[:db/retractEntity eid]])))
           blocks))
 
 (defn- get-file-page

deps/outliner/src/logseq/outliner/core.cljs

@@ -400,7 +400,7 @@
                                   [:db/retract (:db/id block) :block/order]
                                   [:db/retract (:db/id block) :block/page]])
         (let [ids (cons (:db/id this) (ldb/get-block-full-children-ids db (:db/id block)))
-              txs (map (fn [id] [:db.fn/retractEntity id]) ids)
+              txs (map (fn [id] [:db/retractEntity id]) ids)
               page-tx (let [block (d/entity db [:block/uuid block-id])]
                         (when (:block/pre-block? block)
                           (when-let [id (:db/id (:block/page block))]

deps/outliner/src/logseq/outliner/page.cljs

@@ -54,7 +54,7 @@
       (let [blocks (:block/_page page)
             truncate-blocks-tx-data (mapv
                                      (fn [block]
-                                       [:db.fn/retractEntity [:block/uuid (:block/uuid block)]])
+                                       [:db/retractEntity [:block/uuid (:block/uuid block)]])
                                      blocks)]
         ;; TODO: maybe we should add $$$favorites to built-in pages?
         (if (or (ldb/built-in? page) (ldb/hidden? page))
@@ -69,7 +69,7 @@
                                            (d/datoms @conn :avet :logseq.property.history/property (:db/ident page)))))
                 delete-page-tx (concat (db-refs->page page)
                                        delete-property-tx
-                                       [[:db.fn/retractEntity (:db/id page)]])
+                                       [[:db/retractEntity (:db/id page)]])
                 restore-class-parent-tx (->> (filter ldb/class? (:logseq.property.class/_extends page))
                                              (map (fn [p]
                                                     {:db/id (:db/id p)

rebase.md

@@ -26,7 +26,7 @@ conflicts.
 
 Remote tx data is transformed before transact to avoid invalid operations:
 
-- Convert :block/uuid retracts into :db.fn/retractEntity.
+- Convert :block/uuid retracts into :db/retractEntity.
 - Keep only the last :block/parent update per entity.
 - Drop datoms that reference missing entities or missing ref targets.
 - Repair parent cycles by reparenting to the page root.

src/main/frontend/db/model.cljs

@@ -360,7 +360,7 @@ independent of format as format specific heading characters are stripped"
 
 (defn delete-files
   [files]
-  (mapv (fn [path] [:db.fn/retractEntity [:file/path path]]) files))
+  (mapv (fn [path] [:db/retractEntity [:file/path path]]) files))
 
 (defn whiteboard-page?
   "Given a page entity, page object or page name, check if it is a whiteboard page"

src/main/frontend/worker/db_sync.cljs

@@ -279,7 +279,7 @@
     (when-let [conn (client-ops-conn repo)]
       (ldb/transact! conn
                      (mapv (fn [tx-id]
-                             [:db.fn/retractEntity [:db-sync/tx-id tx-id]])
+                             [:db/retractEntity [:db-sync/tx-id tx-id]])
                            tx-ids)))))
 
 (defn get-lookup-id
@@ -294,7 +294,7 @@
   (->> tx-data
        (common-util/distinct-by-last-wins
         (fn [item]
-          (if (and (vector? item) (= 4 (count item))
+          (if (and (vector? item) (= 5 (count item))
                    (contains? #{:block/updated-at :block/title :block/name :block/order} (nth item 2)))
             (take 3 item)
             item)))))
@@ -304,7 +304,7 @@
   (let [sanitized-tx-data (->> tx-data
                                db-normalize/replace-attr-retract-with-retract-entity-v2
                                (remove (fn [item]
-                                         (= :db.fn/retractEntity (first item))
+                                         (= :db/retractEntity (first item))
                                          (contains? local-deleted-ids (get-lookup-id (last item)))))
                                keep-last-update)]
     (when (not= tx-data sanitized-tx-data)
@@ -575,7 +575,9 @@
           local-txs (pending-txs repo)
           reversed-tx-data (->> local-txs
                                 reverse
-                                (mapcat :reversed-tx))
+                                (mapcat :reversed-tx)
+                                db-normalize/replace-attr-retract-with-retract-entity
+                                keep-last-update)
           has-local-changes? (seq reversed-tx-data)
           *remote-tx-report (atom nil)
           *reversed-tx-report (atom nil)
@@ -590,24 +592,24 @@
                             :gen-undo-ops? false
                             :persist-op? false}
                    db @temp-conn
-
                    reversed-tx-report (when has-local-changes?
-                                        (ldb/transact! temp-conn (keep-last-update reversed-tx-data) tx-meta))
+                                        (ldb/transact! temp-conn reversed-tx-data (assoc tx-meta :op :reverse)))
                    _ (reset! *reversed-tx-report reversed-tx-report)
                    ;; 2. transact remote tx-data
                    remote-deleted-blocks (->> tx-data
                                               (keep (fn [item]
-                                                      (when (= :db.fn/retractEntity (first item))
+                                                      (when (= :db/retractEntity (first item))
                                                         (d/entity db (second item))))))
                    remote-deleted-block-ids (set (map :block/uuid remote-deleted-blocks))
-                   remote-tx-report (if has-local-changes?
-                                      (let [tx-data (->> tx-data
-                                                         (remove (fn [item]
-                                                                   (or (= :db.fn/retractEntity (first item))
-                                                                       (contains? remote-deleted-block-ids (get-lookup-id (last item))))))
-                                                         seq)]
-                                        (ldb/transact! temp-conn tx-data tx-meta))
-                                      (ldb/transact! temp-conn tx-data tx-meta))
+                   remote-tx-report (let [tx-meta (assoc tx-meta :op :transact-remote-tx-data)]
+                                      (if has-local-changes?
+                                        (let [tx-data (->> tx-data
+                                                           (remove (fn [item]
+                                                                     (or (= :db/retractEntity (first item))
+                                                                         (contains? remote-deleted-block-ids (get-lookup-id (last item))))))
+                                                           seq)]
+                                          (ldb/transact! temp-conn tx-data tx-meta))
+                                        (ldb/transact! temp-conn tx-data tx-meta)))
                    _ (reset! *remote-tx-report remote-tx-report)
                    remote-received-tx-data (when remote-tx-report (sync-compare/filter-received-tx-data remote-tx-report tx-data))
                    remote-applied-tx-data (when remote-tx-report (sync-compare/filter-applied-tx-data remote-tx-report))]
@@ -635,23 +637,26 @@
                                                 rebased-tx-data (sanitize-tx-data pending-tx-data
                                                                                   (set (map :block/uuid local-deleted-blocks)))]
                                             (when (seq rebased-tx-data)
-                                              (ldb/transact! temp-conn rebased-tx-data tx-meta))))]
-
-                   ;; 4. fix tx data
-                   (let [db @temp-conn
-                         deleted-nodes (keep (fn [id] (d/entity db [:block/uuid id])) deleted-ids)]
-                     (delete-nodes! temp-conn deleted-nodes tx-meta))
-                   (fix-tx! temp-conn remote-tx-report rebase-tx-report tx-meta))))))
+                                              (ldb/transact! temp-conn rebased-tx-data (assoc tx-meta :op :rebase)))))
+                       ;; 4. fix tx data
+                       db @temp-conn
+                       deleted-nodes (keep (fn [id] (d/entity db [:block/uuid id])) deleted-ids)]
+                   (delete-nodes! temp-conn deleted-nodes (assoc tx-meta :op :delete-blocks))
+                   (fix-tx! temp-conn remote-tx-report rebase-tx-report (assoc tx-meta :op :fix))))))
+           {:filter-tx-data (fn [_temp-db tx-data]
+                              (->> tx-data
+                                   db-normalize/replace-attr-retract-with-retract-entity
+                                   keep-last-update))})
           remote-tx-report @*remote-tx-report]
 
       ;; persist rebase tx to client ops
       (when has-local-changes?
         (when-let [tx-data (:tx-data tx-report)]
           (let [remote-tx-data-set (set tx-data*)
-                normalized (normalize-tx-data (:db-after tx-report)
-                                              (or (:db-before remote-tx-report)
-                                                  (:db-after @*reversed-tx-report))
-                                              tx-data)
+                normalized (->> tx-data
+                                (normalize-tx-data (:db-after tx-report)
+                                                   (or (:db-before remote-tx-report)
+                                                       (:db-after @*reversed-tx-report))))
                 normalized-tx-data (remove remote-tx-data-set normalized)
                 reversed-datoms (reverse-tx-data tx-data)]
            ;; (prn :debug :normalized-tx-data normalized-tx-data)

src/main/frontend/worker/pipeline.cljs

@@ -464,7 +464,7 @@
 (defn transact-pipeline
   "Compute extra tx-data and block/refs, should ensure it's a pure function and
   doesn't call `d/transact!` or `ldb/transact!`."
-  [repo {:keys [db-before db-after tx-meta] :as tx-report}]
+  [repo {:keys [db-after tx-meta tx-data] :as tx-report}]
   (when-not (:temp-conn? tx-meta)
     (let [extra-tx-data (compute-extra-tx-data tx-report)
           tx-report* (if (seq extra-tx-data)
@@ -475,27 +475,20 @@
                        tx-report)
           {:keys [pages blocks]} (ds-report/get-blocks-and-pages tx-report*)
           deleted-blocks (outliner-pipeline/filter-deleted-blocks (:tx-data tx-report*))
-          delete-views (mapcat
-                        (fn [item]
-                          (let [block (d/entity db-before (:db/id item))]
-                            (:logseq.property/_view-for block)))
-                        deleted-blocks)
-          deleted-block-ids (set (map :db/id (concat deleted-blocks delete-views)))
+          deleted-block-ids (set (map :db/id deleted-blocks))
           blocks' (remove (fn [b] (deleted-block-ids (:db/id b))) blocks)
           block-refs (when (seq blocks')
                        (rebuild-block-refs repo tx-report* blocks'))
-          tx-id-data (when-not (:temp-conn? tx-meta)
-                       (let [db-after (:db-after tx-report*)
-                             updated-blocks (remove (fn [b] (contains? deleted-block-ids (:db/id b)))
-                                                    (concat pages blocks))
-                             tx-id (get-in tx-report* [:tempids :db/current-tx])]
-                         (keep (fn [b]
-                                 (when-let [db-id (:db/id b)]
-                                   (when (:block/uuid (d/entity db-after db-id))
-                                     {:db/id db-id
-                                      :block/tx-id tx-id}))) updated-blocks)))
-          delete-views-tx-data (map (fn [b] [:db/retractEntity (:db/id b)]) delete-views)
-          block-refs-tx-id-data (concat delete-views-tx-data block-refs tx-id-data)
+          tx-id-data (let [db-after (:db-after tx-report*)
+                           updated-blocks (remove (fn [b] (contains? deleted-block-ids (:db/id b)))
+                                                  (concat pages blocks))
+                           tx-id (get-in tx-report* [:tempids :db/current-tx])]
+                       (keep (fn [b]
+                               (when-let [db-id (:db/id b)]
+                                 (when (:block/uuid (d/entity db-after db-id))
+                                   {:db/id db-id
+                                    :block/tx-id tx-id}))) updated-blocks))
+          block-refs-tx-id-data (concat block-refs tx-id-data)
           replace-tx-report (when (seq block-refs-tx-id-data)
                               (d/with (:db-after tx-report*) block-refs-tx-id-data))
           tx-report' (or replace-tx-report tx-report*)