re-use normalization fns

deps/db/src/logseq/db/common/normalize.cljs

@@ -0,0 +1,53 @@
+(ns logseq.db.common.normalize
+  "Normalize && denormalize eid for sync"
+  (:require [datascript.core :as d]))
+
+(defn normalize-tx-data
+  [db-after db-before tx-data]
+  (->> tx-data
+       (map
+        (fn [[e a v _t added]]
+          (let [e' (if-let [entity (d/entity db-before e)]
+                     (if-let [id (:block/uuid entity)]
+                       [:block/uuid id]
+                       (:db/ident entity))
+                     (- e))
+                v' (if (and (integer? v)
+                            (or (= :db.type/ref (:db/valueType (d/entity db-after a)))
+                                (= :db.type/ref (:db/valueType (d/entity db-before a)))))
+                     (if-let [entity (d/entity db-before v)]
+                       (if-let [id (:block/uuid entity)]
+                         [:block/uuid id]
+                         (:db/ident entity))
+                       (- v))
+                     v)]
+            (when (and (not added)
+                       (number? e')
+                       (neg-int? e'))
+              (throw (ex-info "temp id shouldn't be used in :db/retract"
+                              {:datom [e a v added]
+                               :e' e'})))
+            (if added
+              [:db/add e' a v']
+              [:db/retract e' a v']))))))
+
+(defn- lookup-id?
+  [v]
+  (and (vector? v)
+       (= 2 (count v))
+       (= (first v) :block/uuid)
+       (uuid? (second v))))
+
+(defn de-normalize-tx-data
+  [db tx-data]
+  (keep
+   (fn [[op e a v]]
+     (let [e' (if (lookup-id? e)
+                (:db/id (d/entity db e))
+                e)
+           v' (if (lookup-id? v)
+                (:db/id (d/entity db v))
+                v)]
+       (when (and e' v')
+         [op e' a v'])))
+   tx-data))

deps/worker-sync/src/logseq/worker_sync/worker.cljs

@@ -5,7 +5,7 @@
             [lambdaisland.glogi :as log]
             [lambdaisland.glogi.console :as glogi-console]
             [logseq.db :as ldb]
-            [logseq.db.frontend.schema :as db-schema]
+            [logseq.db.common.normalize :as db-normalize]
             [logseq.worker-sync.common :as common]
             [logseq.worker-sync.cycle :as cycle]
             [logseq.worker-sync.protocol :as protocol]
@@ -106,68 +106,6 @@
           {:attr attr
            :server_values (cycle/server-values-for db tx-data attr)})})
 
-(defn- lookup-id?
-  [v]
-  (and (vector? v)
-       (= 2 (count v))
-       (= (first v) :block/uuid)
-       (uuid? (second v))))
-
-(defn- tempid->lookup-map [tx-data]
-  (reduce
-   (fn [acc [op e a v]]
-     (if (and (= :db/add op) (= :block/uuid a) (uuid? v))
-       (assoc acc e [:block/uuid v])
-       acc))
-   {}
-   tx-data))
-
-(defn- replace-tempids [tx-data]
-  (let [m (tempid->lookup-map tx-data)]
-    (mapv
-     (fn [[op e a v]]
-       (let [e' (get m e e)
-             v' (if (lookup-id? v) v (get m v v))]
-         [op e' a v']))
-     tx-data)))
-
-(defn- normalize-tx-data
-  [db-after db-before tx-data]
-  (->> tx-data
-       (map
-        (fn [[e a v _t added]]
-          (let [e' (if-let [entity (d/entity db-before e)]
-                     (if-let [id (:block/uuid entity)]
-                       [:block/uuid id]
-                       (:db/ident entity))
-                     (- e))
-                v' (if (and (integer? v)
-                            (or (= :db.type/ref (:db/valueType (d/entity db-after a)))
-                                (= :db.type/ref (:db/valueType (d/entity db-before a)))))
-                     (if-let [entity (d/entity db-before v)]
-                       (if-let [id (:block/uuid entity)]
-                         [:block/uuid id]
-                         (:db/ident entity))
-                       (- v))
-                     v)]
-            (if added
-              [:db/add e' a v']
-              [:db/retract e' a v']))))))
-
-(defn- de-normalize-tx-data
-  [db tx-data]
-  (keep
-   (fn [[op e a v]]
-     (let [e' (if (lookup-id? e)
-                (:db/id (d/entity db e))
-                e)
-           v' (if (lookup-id? v)
-                (:db/id (d/entity db v))
-                v)]
-       (when (and e' v')
-         [op e' a v'])))
-   tx-data))
-
 (defn- fix-tx-data
   [db tx-data]
   (if (some (fn [[op _e a v]]
@@ -182,7 +120,7 @@
   (let [sql (.-sql self)
         conn (.-conn self)
         db @conn
-        resolved (de-normalize-tx-data db tx-data)
+        resolved (db-normalize/de-normalize-tx-data db tx-data)
         tx-report (d/with db resolved)
         db' (:db-after tx-report)
         order-fixed (fix-tx-data db' resolved)
@@ -192,7 +130,7 @@
         (prn :debug "cycle detected: " cycle-info)
         (cycle-reject-response db order-fixed cycle-info))
       (let [{:keys [tx-data db-before db-after]} (ldb/transact! conn order-fixed)
-            normalized-data (normalize-tx-data db-after db-before tx-data)
+            normalized-data (db-normalize/normalize-tx-data db-after db-before tx-data)
             new-t (storage/next-t! sql)
             created-at (common/now-ms)
             tx-str (common/write-transit normalized-data)]

src/main/frontend/worker/worker_sync.cljs

@@ -1,13 +1,12 @@
 (ns frontend.worker.worker-sync
   "Simple worker-sync client based on promesa + WebSocket."
-  (:require [cljs.reader :as reader]
-            [clojure.string :as string]
+  (:require [clojure.string :as string]
             [datascript.core :as d]
-            [datascript.impl.entity :as de :refer [Entity]]
             [frontend.worker.state :as worker-state]
             [lambdaisland.glogi :as log]
             [logseq.common.util :as common-util]
             [logseq.db :as ldb]
+            [logseq.db.common.normalize :as db-normalize]
             [logseq.db.sqlite.util :as sqlite-util]
             [promesa.core :as p]))
 
@@ -56,40 +55,13 @@
   (when (ws-open? ws)
     (.send ws (js/JSON.stringify (clj->js message)))))
 
-(defn- normalize-ref [db a value]
-  (if (and (integer? value)
-           (= :db.type/ref (:db/valueType (d/entity db a))))
-    (if-let [id (:block/uuid (d/entity db value))]
-      [:block/uuid id]
-      (throw (ex-info (str "There's no :block/uuid for given refed value: " value)
-                      {:value value})))
-    value))
-
 (defn- normalize-tx-data [db-after db-before tx-data]
   (->> tx-data
-       (remove (fn [[e a v t added]]
+       (remove (fn [[_e a _v _t _added]]
                  (contains? #{:block/tx-id :logseq.property/created-by-ref
                               :logseq.property.embedding/hnsw-label-updated-at} a)))
        (common-util/distinct-by-last-wins (fn [[e a v tx _added]] [e a v tx]))
-       (map
-        (fn [[e a v _t added]]
-          (let [e' (if-let [entity (d/entity db-before e)]
-                     (if-let [id (:block/uuid entity)]
-                       [:block/uuid id]
-                       (:db/ident entity))
-                     (- e))
-                v' (if (and (integer? v)
-                            (or (= :db.type/ref (:db/valueType (d/entity db-after a)))
-                                (= :db.type/ref (:db/valueType (d/entity db-before a)))))
-                     (if-let [entity (d/entity db-before v)]
-                       (if-let [id (:block/uuid entity)]
-                         [:block/uuid id]
-                         (:db/ident entity))
-                       (- v))
-                     v)]
-            (if added
-              [:db/add e' a v']
-              [:db/retract e' a v']))))))
+       (db-normalize/normalize-tx-data db-after db-before)))
 
 (defn- parse-message [raw]
   (try
@@ -101,31 +73,10 @@
   (when (number? t)
     (reset! (:server-t client) t)))
 
-(defn- lookup-id?
-  [v]
-  (and (vector? v)
-       (= 2 (count v))
-       (= (first v) :block/uuid)
-       (uuid? (second v))))
-
-(defn- de-normalized-data
-  [db tx-data]
-  (keep
-   (fn [[op e a v]]
-     (let [e' (if (lookup-id? e)
-                (:db/id (d/entity db e))
-                e)
-           v' (if (lookup-id? v)
-                (:db/id (d/entity db v))
-                v)]
-       (when (and e' v')
-         [op e' a v'])))
-   tx-data))
-
 (defn- apply-remote-tx! [repo tx-data]
   (when-let [conn (worker-state/get-datascript-conn repo)]
     (try
-      (let [tx-data' (de-normalized-data @conn tx-data)]
+      (let [tx-data' (db-normalize/de-normalize-tx-data @conn tx-data)]
         (d/transact! conn tx-data' {:worker-sync/remote? true}))
       (catch :default e
         (log/error :worker-sync/apply-remote-tx-failed {:error e})))))