enhance(rtc): ignore entities if possible when init upload/download

deps/common/src/logseq/common/defkeywords.cljc

@@ -3,7 +3,7 @@
   #?(:cljs (:require-macros [logseq.common.defkeywords])))
 
 (def ^:private *defined-kws (volatile! {}))
-(def *defined-kw->config (volatile! {}))
+(def ^:private *defined-kw->config (volatile! {}))
 
 #_:clj-kondo/ignore
 (defmacro defkeyword
@@ -24,9 +24,12 @@
         (when (not= (:file current-meta) (:file info))
           (vswap! *defined-kws assoc kw current-meta)
           (throw (ex-info "keyword already defined somewhere else" {:kw kw :info info}))))
-      (vswap! *defined-kws assoc kw current-meta)))
-  (let [kw->v (partition 2 keyvals)]
-    `(do
-       (doseq [[kw# config#] '~kw->v]
-         (vswap! *defined-kw->config assoc kw# config#))
-       (vector ~@keyvals))))
+      (vswap! *defined-kws assoc kw current-meta))
+    (let [kw->config (partition 2 keyvals)]
+      (doseq [[kw config] kw->config]
+        (vswap! *defined-kw->config assoc kw config))))
+  `(vector ~@keyvals))
+
+(defmacro get-all-defined-kw->config
+  []
+  `'~(deref *defined-kw->config))

src/main/frontend/worker/rtc/const.cljs

@@ -22,18 +22,27 @@
   ;; {:doc "keyword option for RTC. ignore this *entity* when syncing graph. Default false"}
   )
 
+(def ignore-attrs-when-init-upload
+  (into #{}
+        (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-upload]) kw)))
+        (common-def/get-all-defined-kw->config)))
 
-(def *ignore-attrs-when-init-upload
-  (delay (into #{}
-               (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-upload]) kw)))
-               @common-def/*defined-kw->config)))
+(def ignore-attrs-when-init-download
+  (into #{}
+        (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-download]) kw)))
+        (common-def/get-all-defined-kw->config)))
 
-(def *ignore-attrs-when-init-download
-  (delay (into #{}
-               (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-download]) kw)))
-               @common-def/*defined-kw->config)))
+(def ignore-attrs-when-syncing
+  (into #{}
+        (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-syncing]) kw)))
+        (common-def/get-all-defined-kw->config)))
 
-(def *ignore-attrs-when-syncing
-  (delay (into #{}
-               (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-syncing]) kw)))
-               @common-def/*defined-kw->config)))
+(def ignore-entities-when-init-upload
+  (into #{}
+        (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-entity-when-init-upload]) kw)))
+        (common-def/get-all-defined-kw->config)))
+
+(def ignore-entities-when-init-download
+  (into #{}
+        (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-entity-when-init-download]) kw)))
+        (common-def/get-all-defined-kw->config)))

src/main/frontend/worker/rtc/db_listener.cljs

@@ -125,7 +125,7 @@
   [repo db-before db-after same-entity-datoms-coll e->a->v->add?->t]
   (let [ops (mapcat
              (partial entity-datoms=>ops
-                      db-before db-after e->a->v->add?->t @rtc-const/*ignore-attrs-when-syncing)
+                      db-before db-after e->a->v->add?->t rtc-const/ignore-attrs-when-syncing)
              same-entity-datoms-coll)]
     (when (seq ops)
       (client-op/add-ops repo ops))))

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -78,7 +78,7 @@
     db-schema)))
 
 (defn- export-as-blocks
-  [db & {:keys [ignore-attr-set]}]
+  [db & {:keys [ignore-attr-set ignore-entity-set]}]
   (let [datoms (d/datoms db :eavt)
         db-schema (d/schema db)
         card-many-attrs (schema->card-many-attrs db-schema)
@@ -93,8 +93,12 @@
                                  (not (pos-int? (:v datom))))
                         (throw (ex-info "invalid block data" {:datom datom})))
                       (let [a (:a datom)]
-                        (if (contains? ignore-attr-set a)
-                          r
+                        (cond
+                          (contains? ignore-attr-set a) r
+                          (and (keyword-identical? :db/ident a)
+                               (contains? ignore-entity-set (:v datom)))
+                          (reduced nil)
+                          :else
                           (let [card-many? (contains? card-many-attrs a)
                                 ref? (contains? ref-type-attrs a)]
                             (case [ref? card-many?]
@@ -125,7 +129,9 @@
             (ws-util/send&recv get-ws-create-task {:action "presign-put-temp-s3-obj"})
             (m/sp
               (let [all-blocks (export-as-blocks
-                                @conn :ignore-attr-set @rtc-const/*ignore-attrs-when-init-upload)]
+                                @conn
+                                :ignore-attr-set rtc-const/ignore-attrs-when-init-upload
+                                :ignore-entity-set rtc-const/ignore-entities-when-init-upload)]
                 (ldb/write-transit-str all-blocks)))))]
       (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-data
                                                   :message "uploading data"})
@@ -289,25 +295,32 @@
     (concat id-tx-data blocks-tx-data)))
 
 (defn- remote-all-blocks=>client-blocks+t
-  [all-blocks ignore-attr-set]
+  [all-blocks ignore-attr-set ignore-entity-set]
   (let [{:keys [t blocks]} all-blocks
         card-one-attrs (blocks->card-one-attrs blocks)
         blocks1 (worker-util/profile :convert-card-one-value-from-value-coll
                                      (map (partial convert-card-one-value-from-value-coll card-one-attrs) blocks))
         blocks2 (worker-util/profile :normalize-remote-blocks
                                      (normalized-remote-blocks-coercer blocks1))
-        ;;TODO: remove this, client/schema already converted to :db/cardinality, :db/valueType by remote,
-        ;; and :client/schema should be removed by remote too
-        blocks (map #(dissoc % :client/schema) blocks2)
-        blocks (if (seq ignore-attr-set)
-                 (map (fn [block] (into {} (remove (comp (partial contains? ignore-attr-set) first)) block)) blocks)
-                 blocks)
+        blocks (sequence
+                (comp
+                 ;;TODO: remove this
+                 ;;client/schema already converted to :db/cardinality, :db/valueType by remote,
+                 ;;and :client/schema should be removed by remote too
+                 (map #(dissoc % :client/schema))
+                 (remove (fn [block] (contains? ignore-entity-set (:db/ident block))))
+                 (map (fn [block]
+                        (into {} (remove (comp (partial contains? ignore-attr-set) first)) block))))
+                blocks2)
         blocks (fill-block-fields blocks)]
     {:blocks blocks :t t}))
 
 (defn- new-task--transact-remote-all-blocks
   [all-blocks repo graph-uuid]
-  (let [{:keys [t blocks]} (remote-all-blocks=>client-blocks+t all-blocks @rtc-const/*ignore-attrs-when-init-download)
+  (let [{:keys [t blocks]} (remote-all-blocks=>client-blocks+t
+                            all-blocks
+                            rtc-const/ignore-attrs-when-init-download
+                            rtc-const/ignore-entities-when-init-download)
         [schema-blocks normal-blocks] (blocks->schema-blocks+normal-blocks blocks)
         tx-data (concat
                  (blocks-resolve-temp-id normal-blocks)