enhance: apply defkeywords to built-in classes, add some rtc related keywords

.clj-kondo/hooks/defkeywords.clj

@@ -19,11 +19,9 @@
               :type :defkeywords/invalid-arg))
       :else
       (let [new-node (api/list-node
-                      (map (fn [k]
+                      (map (fn [[kw v]]
                              (api/list-node
-                              [(api/token-node 'logseq.common.defkeywords/defkeyword)
-                               k
-                               (api/token-node "")]))
-                           kws))]
+                              [(api/token-node 'logseq.common.defkeywords/defkeyword) kw v]))
+                           kw->v))]
         {:node (with-meta new-node
                  (meta node))}))))

deps/common/src/logseq/common/defkeywords.cljc

@@ -1,8 +1,9 @@
 (ns logseq.common.defkeywords
-  "Macro 'defkeywords' to def keyword with docstring"
+  "Macro 'defkeywords' to def keyword with config"
   #?(:cljs (:require-macros [logseq.common.defkeywords])))
 
 (def ^:private *defined-kws (volatile! {}))
+(def *defined-kw->config (volatile! {}))
 
 #_:clj-kondo/ignore
 (defmacro defkeyword
@@ -24,4 +25,8 @@
           (vswap! *defined-kws assoc kw current-meta)
           (throw (ex-info "keyword already defined somewhere else" {:kw kw :info info}))))
       (vswap! *defined-kws assoc kw current-meta)))
-  `(vector ~@keyvals))
+  (let [kw->v (partition 2 keyvals)]
+    `(do
+       (doseq [[kw# config#] '~kw->v]
+         (vswap! *defined-kw->config assoc kw# config#))
+       (vector ~@keyvals))))

deps/db/src/logseq/db/frontend/class.cljs

@@ -2,6 +2,7 @@
   "Class related fns for DB graphs and frontend/datascript usage"
   (:require [clojure.set :as set]
             [flatland.ordered.map :refer [ordered-map]]
+            [logseq.common.defkeywords :refer [defkeywords]]
             [logseq.db.frontend.db-ident :as db-ident]
             [logseq.db.sqlite.util :as sqlite-util]))
 
@@ -10,76 +11,77 @@
 
 (def ^:large-vars/data-var built-in-classes
   "Map of built-in classes for db graphs with their :db/ident as keys"
-  (ordered-map
-   :logseq.class/Root {:title "Root Tag"}
-
-   :logseq.class/Tag {:title "Tag"}
-
-   :logseq.class/Property {:title "Property"}
-
-   :logseq.class/Page {:title "Page"}
-
-   :logseq.class/Journal
-   {:title "Journal"
-    :properties {:logseq.property/parent :logseq.class/Page
-                 :logseq.property.journal/title-format "MMM do, yyyy"}}
-
-   :logseq.class/Whiteboard
-   {:title "Whiteboard"
-    :properties {:logseq.property/parent :logseq.class/Page}}
-
-   :logseq.class/Task
-   {:title "Task"
-    :schema {:properties [:logseq.task/status :logseq.task/priority :logseq.task/deadline :logseq.task/scheduled]}}
-
-   :logseq.class/Query
-   {:title "Query"
-    :properties {:logseq.property/icon {:type :tabler-icon :id "search"}}
-    :schema {:properties [:logseq.property/query]}}
-
-   :logseq.class/Card
-   {:title "Card"
-    :schema {:properties [:logseq.property.fsrs/state :logseq.property.fsrs/due]}}
-
-   :logseq.class/Cards
-   {:title "Cards"
-    :properties {:logseq.property/icon {:type :tabler-icon :id "search"}
-                 :logseq.property/parent :logseq.class/Query}}
-
-   :logseq.class/Asset
-   {:title "Asset"
-    :properties {;; :logseq.property/icon {:type :tabler-icon :id "file"}
-                 :logseq.property.class/hide-from-node true
-                 :logseq.property.view/type :logseq.property.view/type.gallery}
-    :schema {:properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]
-             :required-properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]}}
-
-   :logseq.class/Code-block
-   {:title "Code"
-    :properties {:logseq.property.class/hide-from-node true}
-    :schema {:properties [:logseq.property.node/display-type :logseq.property.code/lang]}}
-
-   :logseq.class/Quote-block
-   {:title "Quote"
-    :properties {:logseq.property.class/hide-from-node true}
-    :schema {:properties [:logseq.property.node/display-type]}}
-
-   :logseq.class/Math-block
-   {:title "Math"
-    :properties {:logseq.property.class/hide-from-node true}
-    :schema {:properties [:logseq.property.node/display-type]}}
-
-   :logseq.class/Pdf-annotation
-   {:title "PDF Annotation"
-    :properties {:logseq.property.class/hide-from-node true}
-    :schema {:properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset
-                          :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value
-                          :logseq.property.pdf/hl-type :logseq.property.pdf/hl-image]
-             :required-properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset
-                                   :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value]}}
-
-;; TODO: Add more classes such as :book, :paper, :movie, :music, :project)
-   ))
+  (apply
+   ordered-map
+   (defkeywords
+     :logseq.class/Root {:title "Root Tag"}
+
+     :logseq.class/Tag {:title "Tag"}
+
+     :logseq.class/Property {:title "Property"}
+
+     :logseq.class/Page {:title "Page"}
+
+     :logseq.class/Journal
+     {:title "Journal"
+      :properties {:logseq.property/parent :logseq.class/Page
+                   :logseq.property.journal/title-format "MMM do, yyyy"}}
+
+     :logseq.class/Whiteboard
+     {:title "Whiteboard"
+      :properties {:logseq.property/parent :logseq.class/Page}}
+
+     :logseq.class/Task
+     {:title "Task"
+      :schema {:properties [:logseq.task/status :logseq.task/priority :logseq.task/deadline :logseq.task/scheduled]}}
+
+     :logseq.class/Query
+     {:title "Query"
+      :properties {:logseq.property/icon {:type :tabler-icon :id "search"}}
+      :schema {:properties [:logseq.property/query]}}
+
+     :logseq.class/Card
+     {:title "Card"
+      :schema {:properties [:logseq.property.fsrs/state :logseq.property.fsrs/due]}}
+
+     :logseq.class/Cards
+     {:title "Cards"
+      :properties {:logseq.property/icon {:type :tabler-icon :id "search"}
+                   :logseq.property/parent :logseq.class/Query}}
+
+     :logseq.class/Asset
+     {:title "Asset"
+      :properties {;; :logseq.property/icon {:type :tabler-icon :id "file"}
+                   :logseq.property.class/hide-from-node true
+                   :logseq.property.view/type :logseq.property.view/type.gallery}
+      :schema {:properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]
+               :required-properties [:logseq.property.asset/type :logseq.property.asset/size :logseq.property.asset/checksum]}}
+
+     :logseq.class/Code-block
+     {:title "Code"
+      :properties {:logseq.property.class/hide-from-node true}
+      :schema {:properties [:logseq.property.node/display-type :logseq.property.code/lang]}}
+
+     :logseq.class/Quote-block
+     {:title "Quote"
+      :properties {:logseq.property.class/hide-from-node true}
+      :schema {:properties [:logseq.property.node/display-type]}}
+
+     :logseq.class/Math-block
+     {:title "Math"
+      :properties {:logseq.property.class/hide-from-node true}
+      :schema {:properties [:logseq.property.node/display-type]}}
+
+     :logseq.class/Pdf-annotation
+     {:title "PDF Annotation"
+      :properties {:logseq.property.class/hide-from-node true}
+      :schema {:properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset
+                            :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value
+                            :logseq.property.pdf/hl-type :logseq.property.pdf/hl-image]
+               :required-properties [:logseq.property/ls-type :logseq.property.pdf/hl-color :logseq.property/asset
+                                     :logseq.property.pdf/hl-page :logseq.property.pdf/hl-value]}}
+     ;; TODO: Add more classes such as :book, :paper, :movie, :music, :project)
+     )))
 
 (def page-children-classes
   "Children of :logseq.class/Page"

deps/db/src/logseq/db/frontend/kv_entity.cljs

@@ -0,0 +1,28 @@
+(ns logseq.db.frontend.kv-entity
+  "Define kv entities used by logseq db"
+  (:require [logseq.common.defkeywords :refer [defkeywords]]))
+
+(defkeywords
+  :logseq.kv/db-type {:doc ":kv/value = \"db\" if it's a db-graph"}
+  :logseq.kv/graph-uuid {:doc "store graph-uuid if it's a rtc enabled graph"
+                         :rtc {:rtc/ignore-entity-when-init-upload true
+                               :rtc/ignore-entity-when-init-download true}}
+  :logseq.kv/import-type {:doc ":sqlite-db if import from sqlite.
+                                FIXME: any other values?"
+                          :rtc {:rtc/ignore-entity-when-init-upload true
+                                :rtc/ignore-entity-when-init-download true}}
+  :logseq.kv/imported-at {:doc "graph-import time"
+                          :rtc {:rtc/ignore-entity-when-init-upload true
+                                :rtc/ignore-entity-when-init-download true}}
+  :logseq.kv/graph-local-tx {:doc "local rtc tx-id"
+                             :rtc {:rtc/ignore-entity-when-init-upload true
+                                   :rtc/ignore-entity-when-init-download true}}
+  :logseq.kv/schema-version {:doc "schema version"}
+  :logseq.kv/graph-created-at {:doc "graph create time"}
+  :logseq.kv/latest-code-lang {:doc "latest lang used by code-block"
+                               :rtc {:rtc/ignore-entity-when-init-upload true
+                                     :rtc/ignore-entity-when-init-download true}}
+  :logseq.kv/graph-backup-folder {:doc "graph backup-folder"
+                                  :rtc {:rtc/ignore-entity-when-init-upload true
+                                        :rtc/ignore-entity-when-init-download true}}
+  :logseq.kv/graph-initial-schema-version {:doc "schema-version when graph created"})

deps/db/src/logseq/db/frontend/property.cljs

@@ -52,7 +52,9 @@
    * :attribute - Property keyword that is saved to a datascript attribute outside of :block/properties
    * :queryable? - Boolean for whether property can be queried in the query builder
    * :closed-values - Vec of closed-value maps for properties with choices. Map
-     has keys :value, :db-ident, :uuid and :icon"
+     has keys :value, :db-ident, :uuid and :icon
+   * :rtc - submap for RTC configs. view docs by jumping to keyword definitions.
+  "
   (apply
    ordered-map
    (defkeywords
@@ -435,38 +437,56 @@
              [:logseq.property.view/type.list "List View"]
              [:logseq.property.view/type.gallery "Gallery View"]])
       :properties {:logseq.property/default-value :logseq.property.view/type.table}
-      :queryable? true}
+      :queryable? true
+      :rtc {:rtc/ignore-attr-when-init-upload true
+            :rtc/ignore-attr-when-init-download true
+            :rtc/ignore-attr-when-syncing true}}
 
      :logseq.property.table/sorting {:title "View sorting"
                                      :schema
                                      {:type :coll
                                       :hide? true
-                                      :public? false}}
+                                      :public? false}
+                                     :rtc {:rtc/ignore-attr-when-init-upload true
+                                           :rtc/ignore-attr-when-init-download true
+                                           :rtc/ignore-attr-when-syncing true}}
 
      :logseq.property.table/filters {:title "View filters"
                                      :schema
                                      {:type :coll
                                       :hide? true
-                                      :public? false}}
+                                      :public? false}
+                                     :rtc {:rtc/ignore-attr-when-init-upload true
+                                           :rtc/ignore-attr-when-init-download true
+                                           :rtc/ignore-attr-when-syncing true}}
 
      :logseq.property.table/hidden-columns {:title "View hidden columns"
                                             :schema
                                             {:type :keyword
                                              :cardinality :many
                                              :hide? true
-                                             :public? false}}
+                                             :public? false}
+                                            :rtc {:rtc/ignore-attr-when-init-upload true
+                                                  :rtc/ignore-attr-when-init-download true
+                                                  :rtc/ignore-attr-when-syncing true}}
 
      :logseq.property.table/ordered-columns {:title "View ordered columns"
                                              :schema
                                              {:type :coll
                                               :hide? true
-                                              :public? false}}
+                                              :public? false}
+                                             :rtc {:rtc/ignore-attr-when-init-upload true
+                                                   :rtc/ignore-attr-when-init-download true
+                                                   :rtc/ignore-attr-when-syncing true}}
 
      :logseq.property.table/sized-columns {:title "View columns settings"
                                            :schema
                                            {:type :map
                                             :hide? true
-                                            :public? false}}
+                                            :public? false}
+                                           :rtc {:rtc/ignore-attr-when-init-upload true
+                                                 :rtc/ignore-attr-when-init-download true
+                                                 :rtc/ignore-attr-when-syncing true}}
 
      :logseq.property/view-for {:title "This view belongs to"
                                 :schema
@@ -490,12 +510,18 @@
      :logseq.property.asset/last-visit-page {:title "Last visit page"
                                              :schema {:type :raw-number
                                                       :hide? true
-                                                      :public? false}}
+                                                      :public? false}
+                                             :rtc {:rtc/ignore-attr-when-init-upload true
+                                                   :rtc/ignore-attr-when-init-download true
+                                                   :rtc/ignore-attr-when-syncing true}}
      :logseq.property.asset/remote-metadata {:title "File remote metadata"
                                              :schema
                                              {:type :map
                                               :hide? true
-                                              :public? false}}
+                                              :public? false}
+                                             :rtc {:rtc/ignore-attr-when-init-upload true
+                                                   :rtc/ignore-attr-when-init-download true
+                                                   :rtc/ignore-attr-when-syncing true}}
      :logseq.property.asset/resize-metadata {:title "Asset resize metadata"
                                              :schema {:type :map
                                                       :hide? true

src/main/frontend/core.cljs

@@ -14,6 +14,7 @@
             [frontend.routes :as routes]
             [frontend.spec]
             [logseq.api]
+            [logseq.db.frontend.kv-entity]
             [malli.dev.cljs :as md]
             [reitit.frontend :as rf]
             [reitit.frontend.easy :as rfe]

src/main/frontend/worker/rtc/const.cljs

@@ -1,5 +1,39 @@
 (ns frontend.worker.rtc.const
-  "RTC constants")
+  "RTC constants"
+  (:require [logseq.common.defkeywords :as common-def :refer [defkeywords]]))
 
 (goog-define RTC-E2E-TEST* false)
 (def RTC-E2E-TEST RTC-E2E-TEST*)
+
+(defkeywords
+  :rtc/ignore-attr-when-init-upload
+  {:doc "keyword option for RTC. ignore this *attr* when initial uploading graph. Default false"}
+  :rtc/ignore-attr-when-init-download
+  {:doc "keyword option for RTC. ignore this *attr* when initial downloading graph. Default false"}
+  :rtc/ignore-attr-when-syncing
+  {:doc "keyword option for RTC. ignore this *attr* when syncing graph. Default false"}
+  :rtc/ignore-entity-when-init-upload
+  {:doc "keyword option for RTC. ignore this *entity* when initial uploading graph. Default false"}
+  :rtc/ignore-entity-when-init-download
+  {:doc "keyword option for RTC. ignore this *entity* when initial downloading graph. Default false"}
+
+  ;; only blocks(:block/uuid) will be synced, this option is meaningless for now
+  ;; :rtc/ignore-entity-when-syncing
+  ;; {:doc "keyword option for RTC. ignore this *entity* when syncing graph. Default false"}
+  )
+
+
+(def *ignore-attrs-when-init-upload
+  (delay (into #{}
+               (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-upload]) kw)))
+               @common-def/*defined-kw->config)))
+
+(def *ignore-attrs-when-init-download
+  (delay (into #{}
+               (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-init-download]) kw)))
+               @common-def/*defined-kw->config)))
+
+(def *ignore-attrs-when-syncing
+  (delay (into #{}
+               (keep (fn [[kw config]] (when (get-in config [:rtc :rtc/ignore-attr-when-syncing]) kw)))
+               @common-def/*defined-kw->config)))

src/main/frontend/worker/rtc/full_upload_download_graph.cljs

@@ -78,7 +78,7 @@
     db-schema)))
 
 (defn- export-as-blocks
-  [db]
+  [db & {:keys [ignore-attr-set]}]
   (let [datoms (d/datoms db :eavt)
         db-schema (d/schema db)
         card-many-attrs (schema->card-many-attrs db-schema)
@@ -92,18 +92,19 @@
                       (when (and (contains? #{:block/parent} (:a datom))
                                  (not (pos-int? (:v datom))))
                         (throw (ex-info "invalid block data" {:datom datom})))
-                      (let [a (:a datom)
-                            card-many? (contains? card-many-attrs a)
-                            ref? (contains? ref-type-attrs a)]
-                        (case [ref? card-many?]
-                          [true true]
-                          (update r a conj (str (:v datom)))
-                          [true false]
-                          (assoc r a (str (:v datom)))
-                          [false true]
-                          (update r a conj (ldb/write-transit-str (:v datom)))
-                          [false false]
-                          (assoc r a (ldb/write-transit-str (:v datom))))))
+                      (let [a (:a datom)]
+                        (when-not (contains? ignore-attr-set a)
+                          (let [card-many? (contains? card-many-attrs a)
+                                ref? (contains? ref-type-attrs a)]
+                            (case [ref? card-many?]
+                              [true true]
+                              (update r a conj (str (:v datom)))
+                              [true false]
+                              (assoc r a (str (:v datom)))
+                              [false true]
+                              (update r a conj (ldb/write-transit-str (:v datom)))
+                              [false false]
+                              (assoc r a (ldb/write-transit-str (:v datom))))))))
                     {:db/id (str (:e (first datoms)))}
                     datoms))))
          (map (fn [block]
@@ -122,7 +123,8 @@
             vector
             (ws-util/send&recv get-ws-create-task {:action "presign-put-temp-s3-obj"})
             (m/sp
-              (let [all-blocks (export-as-blocks @conn)]
+              (let [all-blocks (export-as-blocks
+                                @conn :ignore-attr-set @rtc-const/*ignore-attrs-when-init-upload)]
                 (ldb/write-transit-str all-blocks)))))]
       (rtc-log-and-state/rtc-log :rtc.log/upload {:sub-type :upload-data
                                                   :message "uploading data"})
@@ -269,22 +271,24 @@
                               (assoc :db/ident ident)))) ids)
         id-ref-exists? (fn [v] (and (string? v) (or (get id->ident v) (get id->uuid v))))
         blocks-tx-data (map (fn [block]
-                              (->> (map (fn [[k v]]
-                                          (let [v (cond
-                                                    (id-ref-exists? v)
-                                                    (or (get id->ident v) [:block/uuid (get id->uuid v)])
+                              (->> (map
+                                    (fn [[k v]]
+                                      (let [v (cond
+                                                (id-ref-exists? v)
+                                                (or (get id->ident v) [:block/uuid (get id->uuid v)])
 
-                                                    (and (sequential? v) (every? id-ref-exists? v))
-                                                    (map (fn [id] (or (get id->ident id) [:block/uuid (get id->uuid id)])) v)
+                                                (and (sequential? v) (every? id-ref-exists? v))
+                                                (map (fn [id] (or (get id->ident id) [:block/uuid (get id->uuid id)])) v)
 
-                                                    :else
-                                                    v)]
-                                            [k v])) (dissoc block :db/id))
+                                                :else
+                                                v)]
+                                        [k v]))
+                                    (dissoc block :db/id))
                                    (into {}))) blocks)]
     (concat id-tx-data blocks-tx-data)))
 
-(defn- new-task--transact-remote-all-blocks
-  [all-blocks repo graph-uuid]
+(defn- remote-all-blocks=>client-blocks+t
+  [all-blocks ignore-attr-set]
   (let [{:keys [t blocks]} all-blocks
         card-one-attrs (blocks->card-one-attrs blocks)
         blocks1 (worker-util/profile :convert-card-one-value-from-value-coll
@@ -294,7 +298,15 @@
         ;;TODO: remove this, client/schema already converted to :db/cardinality, :db/valueType by remote,
         ;; and :client/schema should be removed by remote too
         blocks (map #(dissoc % :client/schema) blocks2)
-        blocks (fill-block-fields blocks)
+        blocks (if (seq ignore-attr-set)
+                 (map (fn [block] (into {} (remove (comp (partial contains? ignore-attr-set) first)) block)) blocks)
+                 blocks)
+        blocks (fill-block-fields blocks)]
+    {:blocks blocks :t t}))
+
+(defn- new-task--transact-remote-all-blocks
+  [all-blocks repo graph-uuid]
+  (let [{:keys [t blocks]} (remote-all-blocks=>client-blocks+t all-blocks @rtc-const/*ignore-attrs-when-init-download)
         [schema-blocks normal-blocks] (blocks->schema-blocks+normal-blocks blocks)
         tx-data (concat
                  (blocks-resolve-temp-id normal-blocks)
@@ -314,7 +326,7 @@
           (.exportDB worker-obj repo)
           (.transact worker-obj repo init-tx-data {:rtc-download-graph? true
                                                    :gen-undo-ops? false
-                                                     ;; only transact db schema, skip validation to avoid warning
+                                                    ;; only transact db schema, skip validation to avoid warning
                                                    :frontend.worker.pipeline/skip-validate-db? true
                                                    :persist-op? false} (worker-state/get-context))
           (.transact worker-obj repo tx-data {:rtc-download-graph? true